{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "3a2c259b-7b08-4c6b-9002-269d2dc8cfd3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-23T08:36:48.774812Z",
     "iopub.status.busy": "2022-05-23T08:36:48.774180Z",
     "iopub.status.idle": "2022-05-23T08:36:48.782065Z",
     "shell.execute_reply": "2022-05-23T08:36:48.781417Z",
     "shell.execute_reply.started": "2022-05-23T08:36:48.774743Z"
    }
   },
   "source": [
    "# MovieLens-1M CTR模型\n",
    "在个性化推荐场景中，经常会将个性化排序问题可以被建模成 CTR 预估 问题，在这个Notebook中我们展示如何在 MetaSpore 中使用神经网络模型进行 CTR 模型的离线训练、预测、导出，下面以Google提出的 [Wide & Deep](https://arxiv.org/abs/1606.07792) 模型为例。\n",
    "\n",
    "**注意**：在进行下面的测试前，我们假设已经运行完[Data Exploration](./data_exploration.ipynb)中提到的数据准备部分，即将MovieLens中的数据upload到S3云存储中。"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "774591c2-aef7-4ad6-bb93-c6291fd36844",
   "metadata": {},
   "source": [
    "### 1. 特征生成\n",
    "这里我们会进行**训练/测试**集合到划分，生成 CTR 模型使用特征列，并把特征列的说明上传到S3存储中，使之可以被MetaSpore识别。"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "48926b1c-874c-4753-bbb9-82efcb69f7aa",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-23T09:04:33.902139Z",
     "iopub.status.busy": "2022-05-23T09:04:33.901587Z",
     "iopub.status.idle": "2022-05-23T09:04:33.904870Z",
     "shell.execute_reply": "2022-05-23T09:04:33.904221Z",
     "shell.execute_reply.started": "2022-05-23T09:04:33.902111Z"
    },
    "tags": []
   },
   "source": [
    "#### 1.1 数据读取与训练/测试集合划分\n",
    "对于线上的推荐系统而言，我们通常采用时间的划分方式，比如抽取前[-N, -2]天作为训练集，最后1天的数据随机抽取出测试集和验证集。由于我们这里采用的是一个中等规模的电影数据集，数据量并不大，为了处理方便，针对召回、排序的过程并没有特殊的trick，我们这里采用Next-Item的方式：即对一个用户而言，前[-N, -2]个交互序列作为训练集，最后一个交互的电影数据作为测试集：\n",
    "\n",
    "<center>\n",
    "    <img src=\"./resources/split_data.png\" alt=\"split data\" width=\"600\"/>\n",
    "</center>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a8ab7167-5162-40da-a515-b43813f5d6c0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-23T12:05:35.421193Z",
     "iopub.status.busy": "2022-05-23T12:05:35.420929Z",
     "iopub.status.idle": "2022-05-23T12:05:36.023754Z",
     "shell.execute_reply": "2022-05-23T12:05:36.023247Z",
     "shell.execute_reply.started": "2022-05-23T12:05:35.421117Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import metaspore as ms\n",
    "import yaml\n",
    "import argparse\n",
    "import sys\n",
    "import os\n",
    "import subprocess\n",
    "import pyspark\n",
    "from pyspark.sql import SparkSession\n",
    "from pyspark.sql import functions as F\n",
    "from pyspark.sql.types import StructType, StructField, IntegerType, FloatType, LongType, StringType\n",
    "from functools import reduce\n",
    "\n",
    "sys.path.append('../../../../') \n",
    "from python.algos.widedeep_net import WideDeep\n",
    "\n",
    "def load_config(path):\n",
    "    params = dict()\n",
    "    with open(path, 'r') as stream:\n",
    "        params = yaml.load(stream, Loader=yaml.FullLoader)\n",
    "        print('Debug -- load config: ', params)\n",
    "    return params\n",
    "\n",
    "def init_spark():\n",
    "    subprocess.run(['zip', '-r', os.getcwd()+'/python.zip', 'python'], cwd='../../../../')\n",
    "    spark_confs={\n",
    "        \"spark.submit.pyFiles\":\"python.zip\",\n",
    "        \"spark.network.timeout\":\"500\",\n",
    "        \"spark.ui.showConsoleProgress\": \"true\",\n",
    "        \"spark.kubernetes.executor.deleteOnTermination\":\"true\",\n",
    "        # \"spark.kubernetes.namespace\":\"xxx\" # put namespace params here if we want to use computing cluster\n",
    "    }\n",
    "    spark_session = ms.spark.get_session(local=local,\n",
    "                                         app_name=app_name,\n",
    "                                         batch_size=batch_size,\n",
    "                                         worker_count=worker_count,\n",
    "                                         server_count=server_count,\n",
    "                                         worker_memory=worker_memory,\n",
    "                                         server_memory=server_memory,\n",
    "                                         coordinator_memory=coordinator_memory,\n",
    "                                         spark_confs=spark_confs)\n",
    "    sc = spark_session.sparkContext\n",
    "    print('Debug -- spark init')\n",
    "    print('Debug -- version:', sc.version)   \n",
    "    print('Debug -- applicaitonId:', sc.applicationId)\n",
    "    print('Debug -- uiWebUrl:', sc.uiWebUrl)\n",
    "    return spark_session\n",
    "\n",
    "def stop_spark(spark):\n",
    "    print('Debug -- spark stop')\n",
    "    spark.sparkContext.stop()\n",
    "\n",
    "def read_dataset(**kwargs):\n",
    "    ### read movies\n",
    "    movies_schema = StructType([\n",
    "            StructField(\"movie_id\", LongType(), True),\n",
    "            StructField(\"title\", StringType(), True),\n",
    "            StructField(\"genre\", StringType(), True)\n",
    "    ])\n",
    "\n",
    "    movies = spark.read.csv(movies_path, sep='::',inferSchema=False, header=False, schema=movies_schema)\n",
    "    print('Debug -- movies sample:')\n",
    "    movies.show(10)\n",
    "\n",
    "    ### read ratings\n",
    "    ratings_schema = StructType([\n",
    "            StructField(\"user_id\", LongType(), True),\n",
    "            StructField(\"movie_id\", LongType(), True),\n",
    "            StructField(\"rating\", FloatType(), True),\n",
    "            StructField(\"timestamp\", LongType(), True)\n",
    "    ])\n",
    "\n",
    "    ratings = spark.read.csv(ratings_path, sep='::', inferSchema=False, header=False, schema=ratings_schema)\n",
    "    print('Debug -- ratings sample:')\n",
    "    ratings.show(10)\n",
    "\n",
    "    ### read users\n",
    "    users_schema = StructType([\n",
    "            StructField(\"user_id\", LongType(), True),\n",
    "            StructField(\"gender\", StringType(), True),\n",
    "            StructField(\"age\", IntegerType(), True),\n",
    "            StructField(\"occupation\", StringType(), True),\n",
    "            StructField(\"zip\", StringType(), True)\n",
    "    ])\n",
    "\n",
    "    users = spark.read.csv(users_path, sep='::', inferSchema=False, header=False, schema=users_schema)\n",
    "    print('Debug -- users sample:')\n",
    "    users.show(10)\n",
    "\n",
    "    return users, movies, ratings\n",
    "\n",
    "def merge_dataset(users, movies, ratings):\n",
    "    # merge movies, users, ratings\n",
    "    dataset = ratings.join(users, on=ratings.user_id==users.user_id, how='leftouter').drop(users.user_id)\n",
    "    dataset = dataset.join(movies, on=dataset.movie_id==movies.movie_id,how='leftouter').drop(movies.movie_id)\n",
    "    dataset = dataset.select('user_id', \\\n",
    "                            'gender', \\\n",
    "                            'age', \\\n",
    "                            'occupation', \\\n",
    "                            'zip', \\\n",
    "                            'movie_id', \\\n",
    "                            'title', \\\n",
    "                            'genre', \\\n",
    "                            'rating', \\\n",
    "                            'timestamp'\n",
    "                            )\n",
    "    print('Debug -- dataset sample:')\n",
    "    dataset.show(10)\n",
    "    return dataset\n",
    "\n",
    "def split_train_test(dataset):\n",
    "    dataset.registerTempTable('dataset')        \n",
    "    query =\"\"\"\n",
    "    select \n",
    "        *\n",
    "    from\n",
    "    (\n",
    "        select\n",
    "            *,\n",
    "            ROW_NUMBER() OVER(PARTITION BY user_id ORDER BY timestamp DESC) as sample_id\n",
    "        from\n",
    "            dataset\n",
    "    ) ta\n",
    "    where ta.sample_id = 1\n",
    "    order by user_id ASC\n",
    "    \"\"\"\n",
    "    test_dataset = spark.sql(query)\n",
    "    test_dataset = test_dataset.drop('sample_id')\n",
    "    train_dataset = dataset.exceptAll(test_dataset)\n",
    "    return train_dataset, test_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5b25a064-36af-4c73-bcc3-954fbbe8ac14",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-23T12:05:36.025397Z",
     "iopub.status.busy": "2022-05-23T12:05:36.024954Z",
     "iopub.status.idle": "2022-05-23T12:05:45.140909Z",
     "shell.execute_reply": "2022-05-23T12:05:45.140076Z",
     "shell.execute_reply.started": "2022-05-23T12:05:36.025375Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Debug -- Movielens Feature Generation Demo\n",
      "Debug -- load config:  {'app_name': 'MovieLens-1M CTR', 'local': True, 'worker_count': 1, 'server_count': 1, 'batch_size': 128, 'worker_memory': '5G', 'server_memory': '5G', 'coordinator_memory': '5G', 'movies_path': 's3://alphaide-demo/movielens/ml-1m/movies.dat', 'ratings_path': 's3://alphaide-demo/movielens/ml-1m/ratings.dat', 'users_path': 's3://alphaide-demo/movielens/ml-1m/users.dat', 'rank_train_dataset_path': 's3://alphaide-demo/movielens/ml-1m/rank/train.parquet', 'rank_test_dataset_path': 's3://alphaide-demo/movielens/ml-1m/rank/test.parquet', 'column_name_path': 's3://alphaide-demo/movielens/ml-1m/schema/widedeep/column_schema', 'combine_schema_path': 's3://alphaide-demo/movielens/ml-1m/schema/widedeep/deep_combine_column_schema', 'wide_combine_schema_path': 's3://alphaide-demo/movielens/ml-1m/schema/widedeep/wide_combine_column_schema', 'model_in_path': None, 'model_out_path': 's3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/', 'model_export_path': 's3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_export/', 'model_version': '0.1', 'experiment_name': 'movielens_wdl_1m', 'input_label_column_index': 0, 'embedding_size': 10, 'net_dropout': 0.0, 'adam_learning_rate': 1e-05, 'ftrl_alpha': 0.02, 'ftrl_beta': 1.0, 'ftrl_l1': 1.0, 'ftrl_l2': 1.0, 'dnn_hidden_units': [1024, 512, 256, 128]}\n",
      "  adding: python/ (stored 0%)\n",
      "  adding: python/setup.py (deflated 60%)\n",
      "  adding: python/metaspore/ (stored 0%)\n",
      "  adding: python/metaspore/url_utils.py (deflated 40%)\n",
      "  adding: python/metaspore/__init__.py (deflated 69%)\n",
      "  adding: python/metaspore/patching_pickle.py (deflated 72%)\n",
      "  adding: python/metaspore/distributed_trainer.py (deflated 76%)\n",
      "  adding: python/metaspore/ps_launcher.py (deflated 68%)\n",
      "  adding: python/metaspore/two_tower_ranking.py (deflated 80%)\n",
      "  adding: python/metaspore/swing_retrieval.py (deflated 79%)\n",
      "  adding: python/metaspore/distributed_tensor.py (deflated 80%)\n",
      "  adding: python/metaspore/metric.py (deflated 75%)\n",
      "  adding: python/metaspore/agent.py (deflated 77%)\n",
      "  adding: python/metaspore/compat/ (stored 0%)\n",
      "  adding: python/metaspore/compat/__init__.py (deflated 52%)\n",
      "  adding: python/metaspore/compat/ps/ (stored 0%)\n",
      "  adding: python/metaspore/compat/ps/__init__.py (deflated 66%)\n",
      "  adding: python/metaspore/s3_utils.py (deflated 68%)\n",
      "  adding: python/metaspore/stack_trace_utils.py (deflated 43%)\n",
      "  adding: python/metaspore/updater.py (deflated 80%)\n",
      "  adding: python/metaspore/demo.py (deflated 54%)\n",
      "  adding: python/metaspore/two_tower_retrieval.py (deflated 81%)\n",
      "  adding: python/metaspore/experiment.py (deflated 76%)\n",
      "  adding: python/metaspore/job_utils.py (deflated 47%)\n",
      "  adding: python/metaspore/network_utils.py (deflated 52%)\n",
      "  adding: python/metaspore/estimator.py (deflated 82%)\n",
      "  adding: python/metaspore/embedding.py (deflated 83%)\n",
      "  adding: python/metaspore/shell_utils.py (deflated 65%)\n",
      "  adding: python/metaspore/input.py (deflated 55%)\n",
      "  adding: python/metaspore/loss_utils.py (deflated 40%)\n",
      "  adding: python/metaspore/name_utils.py (deflated 37%)\n",
      "  adding: python/metaspore/spark.py (deflated 69%)\n",
      "  adding: python/metaspore/file_utils.py (deflated 69%)\n",
      "  adding: python/metaspore/output.py (deflated 56%)\n",
      "  adding: python/metaspore/nn/ (stored 0%)\n",
      "  adding: python/metaspore/nn/__init__.py (deflated 41%)\n",
      "  adding: python/metaspore/nn/normalization.py (deflated 61%)\n",
      "  adding: python/metaspore/nn/wide_and_deep.py (deflated 75%)\n",
      "  adding: python/metaspore/nn/deep_fm.py (deflated 70%)\n",
      "  adding: python/metaspore/nn/fm.py (deflated 47%)\n",
      "  adding: python/metaspore/cast.py (deflated 68%)\n",
      "  adding: python/metaspore/model.py (deflated 81%)\n",
      "  adding: python/metaspore/initializer.py (deflated 76%)\n",
      "  adding: python/ps/ (stored 0%)\n",
      "  adding: python/ps/job.py (deflated 81%)\n",
      "  adding: python/ps/__init__.py (deflated 42%)\n",
      "  adding: python/algos/ (stored 0%)\n",
      "  adding: python/algos/tuner/ (stored 0%)\n",
      "  adding: python/algos/tuner/base_tuner.py (deflated 70%)\n",
      "  adding: python/algos/__init__.py (deflated 38%)\n",
      "  adding: python/algos/two_tower_retrieval_milvus.py (deflated 81%)\n",
      "  adding: python/algos/pnn_net.py (deflated 70%)\n",
      "  adding: python/algos/dssm_net.py (deflated 78%)\n",
      "  adding: python/algos/item_cf_retrieval.py (deflated 77%)\n",
      "  adding: python/algos/layers.py (deflated 77%)\n",
      "  adding: python/algos/dcn_v2_net.py (deflated 75%)\n",
      "  adding: python/algos/xdeepfm_net.py (deflated 71%)\n",
      "  adding: python/algos/__pycache__/ (stored 0%)\n",
      "  adding: python/algos/__pycache__/layers.cpython-38.pyc (deflated 59%)\n",
      "  adding: python/algos/__pycache__/__init__.cpython-38.pyc (deflated 20%)\n",
      "  adding: python/algos/__pycache__/widedeep_net.cpython-38.pyc (deflated 41%)\n",
      "  adding: python/algos/widedeep_net.py (deflated 68%)\n",
      "  adding: python/algos/simplex/ (stored 0%)\n",
      "  adding: python/algos/simplex/simplex_metric.py (deflated 68%)\n",
      "  adding: python/algos/simplex/simplex_agent.py (deflated 64%)\n",
      "  adding: python/algos/simplex/simplex_net.py (deflated 71%)\n",
      "  adding: python/algos/dcn_net.py (deflated 69%)\n",
      "  adding: python/algos/deepfm_net.py (deflated 69%)\n",
      "  adding: python/algos/autoint_net.py (deflated 70%)\n",
      "  adding: python/scripts/ (stored 0%)\n",
      "  adding: python/scripts/consul/ (stored 0%)\n",
      "  adding: python/scripts/consul/Dockerfile (deflated 44%)\n",
      "  adding: python/scripts/consul/build.sh (deflated 30%)\n",
      "  adding: python/scripts/consul/consul_watch_load.py (deflated 72%)\n",
      "  adding: python/scripts/consul/create_consul_watch.sh (deflated 17%)\n",
      "  adding: python/scripts/preprocessing/ (stored 0%)\n",
      "  adding: python/scripts/preprocessing/test_example_preprocessor.py (deflated 51%)\n",
      "  adding: python/scripts/preprocessing/example_preprocessor.py (deflated 54%)\n",
      "  adding: python/scripts/preprocessing/preprocessor_service.py (deflated 69%)\n",
      "  adding: python/scripts/preprocessing/example_requirements.txt (stored 0%)\n",
      "  adding: python/tests/ (stored 0%)\n",
      "  adding: python/tests/sparse_two_tower_export_demo.py (deflated 73%)\n",
      "  adding: python/tests/dense_xgboost.py (deflated 46%)\n",
      "  adding: python/tests/mnist_mlp_eval.py (deflated 57%)\n",
      "  adding: python/tests/sparse_wdl_export_test.py (deflated 66%)\n",
      "  adding: python/tests/two_tower_retrieval_milvus.py (deflated 81%)\n",
      "  adding: python/tests/requirements.txt (deflated 24%)\n",
      "  adding: python/tests/mnist_mlp.py (deflated 59%)\n",
      "  adding: python/tests/embedding_bag_export.py (deflated 61%)\n",
      "  adding: python/tests/dense_xgboost_grpc_test.py (deflated 44%)\n",
      "  adding: python/tests/sparse_wdl_export_demo.py (deflated 64%)\n",
      "  adding: python/tests/sparse_wdl_grpc_test.py (deflated 52%)\n",
      "  adding: python/tests/sparse_mlp_export_demo.py (deflated 64%)\n",
      "  adding: python/tests/verify_java_tensor_serde.py (deflated 42%)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: An illegal reflective access operation has occurred\n",
      "WARNING: Illegal reflective access by org.apache.spark.unsafe.Platform (file:/usr/local/lib/python3.8/dist-packages/pyspark/jars/spark-unsafe_2.12-3.1.2.jar) to constructor java.nio.DirectByteBuffer(long,int)\n",
      "WARNING: Please consider reporting this to the maintainers of org.apache.spark.unsafe.Platform\n",
      "WARNING: Use --illegal-access=warn to enable warnings of further illegal reflective access operations\n",
      "WARNING: All illegal access operations will be denied in a future release\n",
      "22/05/23 20:05:37 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n",
      "Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties\n",
      "Setting default log level to \"WARN\".\n",
      "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Debug -- spark init\n",
      "Debug -- version: 3.1.2\n",
      "Debug -- applicaitonId: local-1653307538681\n",
      "Debug -- uiWebUrl: http://movielens-102-0:4040\n",
      "Debug -- movies sample:\n",
      "+--------+--------------------+--------------------+\n",
      "|movie_id|               title|               genre|\n",
      "+--------+--------------------+--------------------+\n",
      "|       1|    Toy Story (1995)|Animation|Childre...|\n",
      "|       2|      Jumanji (1995)|Adventure|Childre...|\n",
      "|       3|Grumpier Old Men ...|      Comedy|Romance|\n",
      "|       4|Waiting to Exhale...|        Comedy|Drama|\n",
      "|       5|Father of the Bri...|              Comedy|\n",
      "|       6|         Heat (1995)|Action|Crime|Thri...|\n",
      "|       7|      Sabrina (1995)|      Comedy|Romance|\n",
      "|       8| Tom and Huck (1995)|Adventure|Children's|\n",
      "|       9| Sudden Death (1995)|              Action|\n",
      "|      10|    GoldenEye (1995)|Action|Adventure|...|\n",
      "+--------+--------------------+--------------------+\n",
      "only showing top 10 rows\n",
      "\n",
      "Debug -- ratings sample:\n",
      "+-------+--------+------+---------+\n",
      "|user_id|movie_id|rating|timestamp|\n",
      "+-------+--------+------+---------+\n",
      "|      1|    1193|   5.0|978300760|\n",
      "|      1|     661|   3.0|978302109|\n",
      "|      1|     914|   3.0|978301968|\n",
      "|      1|    3408|   4.0|978300275|\n",
      "|      1|    2355|   5.0|978824291|\n",
      "|      1|    1197|   3.0|978302268|\n",
      "|      1|    1287|   5.0|978302039|\n",
      "|      1|    2804|   5.0|978300719|\n",
      "|      1|     594|   4.0|978302268|\n",
      "|      1|     919|   4.0|978301368|\n",
      "+-------+--------+------+---------+\n",
      "only showing top 10 rows\n",
      "\n",
      "Debug -- users sample:\n",
      "+-------+------+---+----------+-----+\n",
      "|user_id|gender|age|occupation|  zip|\n",
      "+-------+------+---+----------+-----+\n",
      "|      1|     F|  1|        10|48067|\n",
      "|      2|     M| 56|        16|70072|\n",
      "|      3|     M| 25|        15|55117|\n",
      "|      4|     M| 45|         7|02460|\n",
      "|      5|     M| 25|        20|55455|\n",
      "|      6|     F| 50|         9|55117|\n",
      "|      7|     M| 35|         1|06810|\n",
      "|      8|     M| 25|        12|11413|\n",
      "|      9|     M| 25|        17|61614|\n",
      "|     10|     F| 35|         1|95370|\n",
      "+-------+------+---+----------+-----+\n",
      "only showing top 10 rows\n",
      "\n",
      "Debug -- dataset sample:\n",
      "+-------+------+---+----------+-----+--------+--------------------+--------------------+------+---------+\n",
      "|user_id|gender|age|occupation|  zip|movie_id|               title|               genre|rating|timestamp|\n",
      "+-------+------+---+----------+-----+--------+--------------------+--------------------+------+---------+\n",
      "|      1|     F|  1|        10|48067|    1193|One Flew Over the...|               Drama|   5.0|978300760|\n",
      "|      1|     F|  1|        10|48067|     661|James and the Gia...|Animation|Childre...|   3.0|978302109|\n",
      "|      1|     F|  1|        10|48067|     914| My Fair Lady (1964)|     Musical|Romance|   3.0|978301968|\n",
      "|      1|     F|  1|        10|48067|    3408|Erin Brockovich (...|               Drama|   4.0|978300275|\n",
      "|      1|     F|  1|        10|48067|    2355|Bug's Life, A (1998)|Animation|Childre...|   5.0|978824291|\n",
      "|      1|     F|  1|        10|48067|    1197|Princess Bride, T...|Action|Adventure|...|   3.0|978302268|\n",
      "|      1|     F|  1|        10|48067|    1287|      Ben-Hur (1959)|Action|Adventure|...|   5.0|978302039|\n",
      "|      1|     F|  1|        10|48067|    2804|Christmas Story, ...|        Comedy|Drama|   5.0|978300719|\n",
      "|      1|     F|  1|        10|48067|     594|Snow White and th...|Animation|Childre...|   4.0|978302268|\n",
      "|      1|     F|  1|        10|48067|     919|Wizard of Oz, The...|Adventure|Childre...|   4.0|978301368|\n",
      "+-------+------+---+----------+-----+--------+--------------------+--------------------+------+---------+\n",
      "only showing top 10 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print('Debug -- Movielens Feature Generation Demo')\n",
    "params = load_config('./2-ctr_prediction.yaml')\n",
    "locals().update(params)\n",
    "spark = init_spark()\n",
    "\n",
    "users, movies, ratings = read_dataset(**params)\n",
    "merged_dataset = merge_dataset(users, movies, ratings)\n",
    "train_dataset, test_dataset = split_train_test(merged_dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0e43b910-bfa4-4c4e-a5cc-f089f35fb16a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-23T09:50:54.313142Z",
     "iopub.status.busy": "2022-05-23T09:50:54.312594Z",
     "iopub.status.idle": "2022-05-23T09:50:54.317010Z",
     "shell.execute_reply": "2022-05-23T09:50:54.316254Z",
     "shell.execute_reply.started": "2022-05-23T09:50:54.313118Z"
    }
   },
   "source": [
    "#### 1.2 正负样本划分并将训练/测试数据存储到S3\n",
    "我们知道MovieLens数据只有用户评分，并没有典型的推荐场景那种（样本, 曝光，点击）这样的用户反馈数据。在这里，我们参考[AutoInt: Automatic Feature Interaction Learning via Self-Attentive Neural Networks](https://arxiv.org/abs/1810.11921) 文章中，对数据集进行划分的方式：\n",
    "\n",
    "| 电影评分 | 样本数量  | 划分类型  |\n",
    "|---|---|---|\n",
    "| >3  |  575281  | 正样本  |\n",
    "| =3  |  261197  | 负样本  | \n",
    "|  < 3 | 163731  | 负样本  |\n",
    "\n",
    "经过以上的数据划分，我们就可以对CTR模型的正负样本进行划分。以上处理方式较为简单，训练出的模型相对于真实场景来说预测较为容易，因为一般观看电影其实已经是用户对电影比较感兴趣了。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4d37b46b-080e-4d80-81e4-2bc9a8eaffc1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-23T12:05:45.142977Z",
     "iopub.status.busy": "2022-05-23T12:05:45.142339Z",
     "iopub.status.idle": "2022-05-23T12:05:45.153236Z",
     "shell.execute_reply": "2022-05-23T12:05:45.152722Z",
     "shell.execute_reply.started": "2022-05-23T12:05:45.142947Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import time\n",
    "def prepare_rank_train(spark, dataset, verbose=True, mode='train'):\n",
    "    start = time.time()\n",
    "    dataset = dataset.filter(dataset['rating'] != 3)\n",
    "    dataset = dataset.select(F.when(F.col('rating')> 3, '1', ).otherwise('0').alias('label'), '*')\n",
    "    dataset = dataset.withColumn('rand', F.rand(seed=100)).orderBy('rand')\n",
    "    dataset = dataset.drop('rand', 'timestamp', 'rating')\n",
    "    dataset = dataset.select(*(F.col(c).cast('string').alias(c) for c in dataset.columns))\n",
    "    print('Debug -- prepare_rank_train cost time:', time.time() - start)\n",
    "    if verbose:\n",
    "        print('Debug -- rank %s sample size:'% mode, dataset.count())\n",
    "        print('Debug -- rank %s data types:'% mode, dataset.dtypes)\n",
    "        print('Debug -- rank %s sample:'% mode)\n",
    "        dataset.show(10)\n",
    "        print('Debug -- prepare_rank_train total cost time:', time.time() - start)\n",
    "    return dataset\n",
    "\n",
    "def prepare_rank_test(spark, dataset, verbose=True):\n",
    "    return prepare_rank_train(spark, dataset, verbose=verbose, mode='test')\n",
    "\n",
    "def write_dataset_to_s3(rank_train_dataset, rank_test_dataset, **kwargs):\n",
    "    start = time.time()\n",
    "    rank_train_dataset.write.parquet(rank_train_dataset_path, mode=\"overwrite\")\n",
    "    print('Debug -- write_dataset_to_s3 train cost time:', time.time() - start)\n",
    "    start = time.time()\n",
    "    rank_test_dataset.write.parquet(rank_test_dataset_path, mode=\"overwrite\")\n",
    "    print('Debug -- write_dataset_to_s3 test cost time:', time.time() - start)\n",
    "    return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "df8bb2d3-9651-4e84-8d12-e5081567d15d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-23T12:05:45.154225Z",
     "iopub.status.busy": "2022-05-23T12:05:45.154026Z",
     "iopub.status.idle": "2022-05-23T12:07:12.693469Z",
     "shell.execute_reply": "2022-05-23T12:07:12.692973Z",
     "shell.execute_reply.started": "2022-05-23T12:05:45.154205Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Debug -- prepare_rank_train cost time: 0.18449735641479492\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Debug -- rank train sample size: 734372\n",
      "Debug -- rank train data types: [('label', 'string'), ('user_id', 'string'), ('gender', 'string'), ('age', 'string'), ('occupation', 'string'), ('zip', 'string'), ('movie_id', 'string'), ('title', 'string'), ('genre', 'string')]\n",
      "Debug -- rank train sample:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+-------+------+---+----------+-----+--------+--------------------+--------------------+\n",
      "|label|user_id|gender|age|occupation|  zip|movie_id|               title|               genre|\n",
      "+-----+-------+------+---+----------+-----+--------+--------------------+--------------------+\n",
      "|    1|   2305|     M| 50|        20|48104|    1641|Full Monty, The (...|              Comedy|\n",
      "|    0|   5605|     F| 18|         2|95008|    3499|       Misery (1990)|              Horror|\n",
      "|    1|   2387|     M| 25|        17|32224|    1120|People vs. Larry ...|               Drama|\n",
      "|    1|   1624|     M| 25|         0|06810|      10|    GoldenEye (1995)|Action|Adventure|...|\n",
      "|    1|   3377|     M| 25|        17|03570|    1200|       Aliens (1986)|Action|Sci-Fi|Thr...|\n",
      "|    1|   2427|     F| 25|        14|94010|    2997|Being John Malkov...|              Comedy|\n",
      "|    1|    454|     M| 25|        20|55092|    3160|     Magnolia (1999)|               Drama|\n",
      "|    1|    339|     M| 50|         7|80207|     110|   Braveheart (1995)|    Action|Drama|War|\n",
      "|    1|   1586|     M| 45|         7|77351|    3481|High Fidelity (2000)|              Comedy|\n",
      "|    1|   2077|     M| 18|         0|55112|    3784|     Kid, The (2000)|              Comedy|\n",
      "+-----+-------+------+---+----------+-----+--------+--------------------+--------------------+\n",
      "only showing top 10 rows\n",
      "\n",
      "Debug -- prepare_rank_train total cost time: 24.345274925231934\n",
      "Debug -- prepare_rank_train cost time: 0.08759951591491699\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Debug -- rank test sample size: 4640\n",
      "Debug -- rank test data types: [('label', 'string'), ('user_id', 'string'), ('gender', 'string'), ('age', 'string'), ('occupation', 'string'), ('zip', 'string'), ('movie_id', 'string'), ('title', 'string'), ('genre', 'string')]\n",
      "Debug -- rank test sample:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----+-------+------+---+----------+-----+--------+--------------------+--------------------+\n",
      "|label|user_id|gender|age|occupation|  zip|movie_id|               title|               genre|\n",
      "+-----+-------+------+---+----------+-----+--------+--------------------+--------------------+\n",
      "|    1|   5966|     F| 35|         9|10021|     497|Much Ado About No...|      Comedy|Romance|\n",
      "|    0|   4173|     F| 18|        11|29063|      95| Broken Arrow (1996)|     Action|Thriller|\n",
      "|    1|   1718|     M| 18|         4|43235|    1396|     Sneakers (1992)|  Crime|Drama|Sci-Fi|\n",
      "|    0|    962|     F| 25|         1|80020|    3896|Way of the Gun, T...|      Crime|Thriller|\n",
      "|    1|    937|     M| 25|        15|60513|     750|Dr. Strangelove o...|          Sci-Fi|War|\n",
      "|    1|   3212|     F| 25|        17|11215|    2959|   Fight Club (1999)|               Drama|\n",
      "|    1|   4368|     M| 18|        17|22043|    1645|Devil's Advocate,...|Crime|Horror|Myst...|\n",
      "|    1|   2678|     M| 25|         6|49707|    2396|Shakespeare in Lo...|      Comedy|Romance|\n",
      "|    1|     63|     M| 18|         4|54902|     541| Blade Runner (1982)|    Film-Noir|Sci-Fi|\n",
      "|    1|    432|     M| 45|        16|55306|    3004|Bachelor, The (1999)|      Comedy|Romance|\n",
      "+-----+-------+------+---+----------+-----+--------+--------------------+--------------------+\n",
      "only showing top 10 rows\n",
      "\n",
      "Debug -- prepare_rank_train total cost time: 9.023691177368164\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Debug -- write_dataset_to_s3 train cost time: 32.89811635017395\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Debug -- write_dataset_to_s3 test cost time: 21.252429008483887\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rank_train_dataset = prepare_rank_train(spark, train_dataset)\n",
    "rank_test_dataset = prepare_rank_test(spark, test_dataset)\n",
    "write_dataset_to_s3(rank_train_dataset, rank_test_dataset, **params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4e7389aa-1fb9-431d-9231-222fa86a4c95",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-23T12:07:12.694620Z",
     "iopub.status.busy": "2022-05-23T12:07:12.694266Z",
     "iopub.status.idle": "2022-05-23T12:07:13.508452Z",
     "shell.execute_reply": "2022-05-23T12:07:13.507870Z",
     "shell.execute_reply.started": "2022-05-23T12:07:12.694600Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                           PRE test.parquet/\n",
      "                           PRE train.parquet/\n"
     ]
    }
   ],
   "source": [
    "!aws s3 ls s3://alphaide-demo/movielens/ml-1m/rank/"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c4592f15-e5a5-4add-8635-7304b48840d4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-23T10:17:27.261287Z",
     "iopub.status.busy": "2022-05-23T10:17:27.260747Z",
     "iopub.status.idle": "2022-05-23T10:17:27.263732Z",
     "shell.execute_reply": "2022-05-23T10:17:27.263314Z",
     "shell.execute_reply.started": "2022-05-23T10:17:27.261265Z"
    },
    "tags": []
   },
   "source": [
    "#### 1.4 上传特征列与特征组合的Schema\n",
    "上传特征列与特征组合的Schema到S3云存储，这里有3点需要说明:\n",
    " * 在描述文件 `column_schema` 中，我们把每列的名字进行罗列;\n",
    " * 在 `combine_column_schema` 中是模型中真实使用的特征，每个`feas`占用一行；\n",
    " * 如果模型中使用组合特征，只需要在combine_column_schema文件中加入`feas1#feas2#feas3`即可。 \n",
    "\n",
    "我们这里为了演示，只用了user id和movie id两个特征，特征描述列："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7f9d8138-bedd-483a-bd46-1bb2343cf860",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-23T12:07:13.510165Z",
     "iopub.status.busy": "2022-05-23T12:07:13.509602Z",
     "iopub.status.idle": "2022-05-23T12:07:13.921149Z",
     "shell.execute_reply": "2022-05-23T12:07:13.920562Z",
     "shell.execute_reply.started": "2022-05-23T12:07:13.510130Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 label\n",
      "1 user_id\n",
      "2 gender\n",
      "3 age\n",
      "4 occupation\n",
      "5 zip\n",
      "6 movie_id\n",
      "7 title\n",
      "8 genre"
     ]
    }
   ],
   "source": [
    "!cat ./schema/column_schema"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bad46607-6ee7-43d9-9d51-7e55885137db",
   "metadata": {},
   "source": [
    "模型wide部分使用的特征："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "0bc7417f-0b0e-4dd2-abfa-1f53a2aa879d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-23T12:07:13.923748Z",
     "iopub.status.busy": "2022-05-23T12:07:13.923437Z",
     "iopub.status.idle": "2022-05-23T12:07:14.343728Z",
     "shell.execute_reply": "2022-05-23T12:07:14.343141Z",
     "shell.execute_reply.started": "2022-05-23T12:07:13.923700Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "user_id\n",
      "movie_id"
     ]
    }
   ],
   "source": [
    "!cat ./schema/wide_combine_column_schema"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "97991157-9245-46ae-8bc9-9c16a2bea745",
   "metadata": {},
   "source": [
    "模型deep部分使用的特征："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "fb61c9e8-13a7-449f-9e30-32cf61e434ee",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-23T12:07:14.345249Z",
     "iopub.status.busy": "2022-05-23T12:07:14.344738Z",
     "iopub.status.idle": "2022-05-23T12:07:14.760724Z",
     "shell.execute_reply": "2022-05-23T12:07:14.760113Z",
     "shell.execute_reply.started": "2022-05-23T12:07:14.345226Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "user_id\n",
      "movie_id"
     ]
    }
   ],
   "source": [
    "!cat ./schema/deep_combine_column_schema"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d68ea622-c3dd-4c75-b66e-f96daf61bd97",
   "metadata": {},
   "source": [
    "上传数据："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "10dd5252-8a6d-42eb-9fa4-28dd61b737af",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-23T12:07:14.762053Z",
     "iopub.status.busy": "2022-05-23T12:07:14.761753Z",
     "iopub.status.idle": "2022-05-23T12:07:15.577878Z",
     "shell.execute_reply": "2022-05-23T12:07:15.577297Z",
     "shell.execute_reply.started": "2022-05-23T12:07:14.762012Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "upload: schema/.ipynb_checkpoints/wide_combine_column_schema-checkpoint to s3://alphaide-demo/movielens/ml-1m/schema/widedeep/.ipynb_checkpoints/wide_combine_column_schema-checkpoint\n",
      "upload: schema/.ipynb_checkpoints/deep_combine_column_schema-checkpoint to s3://alphaide-demo/movielens/ml-1m/schema/widedeep/.ipynb_checkpoints/deep_combine_column_schema-checkpoint\n",
      "upload: schema/.ipynb_checkpoints/column_schema-checkpoint to s3://alphaide-demo/movielens/ml-1m/schema/widedeep/.ipynb_checkpoints/column_schema-checkpoint\n",
      "upload: schema/wide_combine_column_schema to s3://alphaide-demo/movielens/ml-1m/schema/widedeep/wide_combine_column_schema\n",
      "upload: schema/column_schema to s3://alphaide-demo/movielens/ml-1m/schema/widedeep/column_schema\n",
      "upload: schema/deep_combine_column_schema to s3://alphaide-demo/movielens/ml-1m/schema/widedeep/deep_combine_column_schema\n"
     ]
    }
   ],
   "source": [
    "!aws s3 cp --recursive schema/  s3://alphaide-demo/movielens/ml-1m/schema/widedeep/"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2e735c84-1e52-4241-ac62-e5abf7b49769",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-23T10:34:40.359636Z",
     "iopub.status.busy": "2022-05-23T10:34:40.359069Z",
     "iopub.status.idle": "2022-05-23T10:34:40.363173Z",
     "shell.execute_reply": "2022-05-23T10:34:40.362582Z",
     "shell.execute_reply.started": "2022-05-23T10:34:40.359611Z"
    }
   },
   "source": [
    "### 2. 模型训练\n",
    "我们使用 Wide & Deep 模型训练、测试并导出 CTR 模型，这里使用`Cross Entropy` 作为模型的 Loss 。我们在 `MetaSpore` 代码库中已经定义了 Wide & Deep 的模型结构，在调用模型的时候，我们只需要对特征、学习率、训练轮数等一些超参数进行定义，就可以进行模型训练了。\n",
    "\n",
    "#### 2.1 定义训练过程和参数\n",
    "我们定义我们的训练所需要的一些函数，并设置超参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "21c247f7-783a-488b-8923-eb4a43296d54",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-23T12:07:15.579392Z",
     "iopub.status.busy": "2022-05-23T12:07:15.578891Z",
     "iopub.status.idle": "2022-05-23T12:07:15.584969Z",
     "shell.execute_reply": "2022-05-23T12:07:15.584518Z",
     "shell.execute_reply.started": "2022-05-23T12:07:15.579367Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def train(spark, train_dataset, **model_params):\n",
    "    ## init wide and deep model\n",
    "    module = WideDeep(use_wide=True,\n",
    "                      wide_embedding_dim=embedding_size,\n",
    "                      deep_embedding_dim=embedding_size,\n",
    "                      wide_column_name_path=column_name_path,\n",
    "                      wide_combine_schema_path=wide_combine_schema_path,\n",
    "                      deep_column_name_path=column_name_path,\n",
    "                      deep_combine_schema_path=combine_schema_path,\n",
    "                      dnn_hidden_units=dnn_hidden_units,\n",
    "                      ftrl_l1=ftrl_l1,\n",
    "                      ftrl_l2=ftrl_l2,\n",
    "                      ftrl_alpha=ftrl_alpha,\n",
    "                      ftrl_beta=ftrl_beta)\n",
    "    \n",
    "    estimator = ms.PyTorchEstimator(module=module,\n",
    "                                    worker_count=worker_count,\n",
    "                                    server_count=server_count,\n",
    "                                    model_out_path=model_out_path,\n",
    "                                    model_export_path=model_export_path,\n",
    "                                    model_version=model_version,\n",
    "                                    experiment_name=experiment_name,\n",
    "                                    input_label_column_index=input_label_column_index,\n",
    "                                    metric_update_interval=100)\n",
    "    model = estimator.fit(train_dataset)\n",
    "     ## dnn learning rate\n",
    "    estimator.updater = ms.AdamTensorUpdater(adam_learning_rate)\n",
    "    return model\n",
    "\n",
    "def transform(spark, model, test_dataset):\n",
    "    test_result = model.transform(test_dataset)\n",
    "    print('Debug -- test result sample:')\n",
    "    test_result.show(20)\n",
    "    return test_result\n",
    "\n",
    "def evaluate(spark, test_result):\n",
    "    evaluator = pyspark.ml.evaluation.BinaryClassificationEvaluator()\n",
    "    auc = evaluator.evaluate(test_result)\n",
    "    return auc"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c47fecd-a3c0-4cc5-a531-7067a49a48e1",
   "metadata": {},
   "source": [
    "#### 2.2 模型训练与导出\n",
    "当模型训练完成后，模型会以 `ONNX` 的格式导出到实现定义的 `model_export_path` 中，方便后续 `MetaSpore Serving` 加载。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "5c69d0c2-e203-4e4a-92ca-14fb72e4ad6b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-23T12:07:15.585929Z",
     "iopub.status.busy": "2022-05-23T12:07:15.585742Z",
     "iopub.status.idle": "2022-05-23T12:11:37.435612Z",
     "shell.execute_reply": "2022-05-23T12:11:37.435036Z",
     "shell.execute_reply.started": "2022-05-23T12:07:15.585913Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Get aws endpoint from env: obs.cn-southwest-2.myhuaweicloud.com\n",
      "[WARN] 2022-05-23 12:07:15.719 STSAssumeRoleWithWebIdentityCredentialsProvider [140013911443264] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:07:15.719] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/column_schema, read_only true\n",
      "[2022-05-23 20:07:15.728] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/column_schema with total length 78\n",
      "[2022-05-23 20:07:15.731] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/column_schema with size 78 at position 0 larger than total size: 78, change size to 78\n",
      "[2022-05-23 20:07:15.735] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/column_schema reached end 78\n",
      "[2022-05-23 20:07:15.736] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/column_schema reached end 78\n",
      "\u001b[32mloaded combine schema from\u001b[m \u001b[32mcolumn name file \u001b[m's3://alphaide-demo/movielens/ml-1m/schema/widedeep/column_schema' \u001b[32mand combine schema file \u001b[m's3://alphaide-demo/movielens/ml-1m/schema/widedeep/wide_combine_column_schema'\n",
      "[WARN] 2022-05-23 12:07:15.737 STSAssumeRoleWithWebIdentityCredentialsProvider [140013911443264] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:07:15.737] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/wide_combine_column_schema, read_only true\n",
      "[2022-05-23 20:07:15.744] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/wide_combine_column_schema with total length 16\n",
      "[2022-05-23 20:07:15.747] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/wide_combine_column_schema with size 16 at position 0 larger than total size: 16, change size to 16\n",
      "[2022-05-23 20:07:15.761] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/wide_combine_column_schema reached end 16\n",
      "[2022-05-23 20:07:15.762] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/wide_combine_column_schema reached end 16\n",
      "[WARN] 2022-05-23 12:07:15.785 STSAssumeRoleWithWebIdentityCredentialsProvider [140013911443264] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:07:15.785] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/column_schema, read_only true\n",
      "[2022-05-23 20:07:15.792] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/column_schema with total length 78\n",
      "[2022-05-23 20:07:15.793] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/column_schema with size 78 at position 0 larger than total size: 78, change size to 78\n",
      "[2022-05-23 20:07:15.797] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/column_schema reached end 78\n",
      "[2022-05-23 20:07:15.798] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/column_schema reached end 78\n",
      "[WARN] 2022-05-23 12:07:15.798 STSAssumeRoleWithWebIdentityCredentialsProvider [140013911443264] Token file must be specified to use STS Assu\u001b[32mloaded combine schema from\u001b[m \u001b[32mcolumn name file \u001b[m's3://alphaide-demo/movielens/ml-1m/schema/widedeep/column_schema' \u001b[32mand combine schema file \u001b[m's3://alphaide-demo/movielens/ml-1m/schema/widedeep/deep_combine_column_schema'\n",
      "meRole web identity creds provider.\n",
      "[2022-05-23 20:07:15.798] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/deep_combine_column_schema, read_only true\n",
      "[2022-05-23 20:07:15.808] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/deep_combine_column_schema with total length 16\n",
      "[2022-05-23 20:07:15.810] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/deep_combine_column_schema with size 16 at position 0 larger than total size: 16, change size to 16\n",
      "[2022-05-23 20:07:15.814] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/deep_combine_column_schema reached end 16\n",
      "[2022-05-23 20:07:15.815] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/deep_combine_column_schema reached end 16\n",
      "[2022-05-23 20:07:15.911] [info] PS job with coordinator address 172.16.0.198:39951 started.\n",
      "[2022-05-23 20:07:15.911] [info] PSRunner::RunPS: pid: 7895, tid: 8373, thread: 0x7f5722193700\n",
      "[2022-05-23 20:07:15.911] [info] PSRunner::RunPSCoordinator: pid: 7895, tid: 8373, thread: 0x7f5722193700\n",
      "[2022-05-23 20:07:15.912] [info] ActorProcess::Receiving: Coordinator pid: 7895, tid: 8376, thread: 0x7f5720990700\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2022-05-23 20:07:17.162] [info] PS job with coordinator address 172.16.0.198:39951 started.\n",
      "[2022-05-23 20:07:17.162] [info] PSRunner::RunPS: pid: 8383, tid: 8390, thread: 0x7fa6b266b700\n",
      "[2022-05-23 20:07:17.162] [info] PSRunner::RunPSWorker: pid: 8383, tid: 8390, thread: 0x7fa6b266b700\n",
      "\u001b[38;5;046mps agent registered for process 8383 thread 0x7fa6d4083740\u001b[m\n",
      "[2022-05-23 20:07:17.163] [info] ActorProcess::Receiving: Worker pid: 8383, tid: 8393, thread: 0x7fa6b0e68700\n",
      "[2022-05-23 20:07:17.164] [info] PS job with coordinator address 172.16.0.198:39951 started.\n",
      "[2022-05-23 20:07:17.164] [info] PSRunner::RunPS: pid: 8382, tid: 8382, thread: 0x7fa6d4083740\n",
      "[2022-05-23 20:07:17.164] [info] PSRunner::RunPSServer: pid: 8382, tid: 8382, thread: 0x7fa6d4083740\n",
      "[2022-05-23 20:07:17.165] [info] ActorProcess::Receiving: Server pid: 8382, tid: 8396, thread: 0x7fa6b1669700\n",
      "[2022-05-23 20:07:17.166] [info] S[0]:10 has connected to others.\n",
      "PS Server node \u001b[38;5;196mS[0]:10\u001b[m is ready.\n",
      "[2022-05-23 20:07:17.166] [info] W[0]:12 has connected to others.\n",
      "PS Worker node \u001b[38;5;051mW[0]:12\u001b[m is ready.\n",
      "\u001b[38;5;196minit sparse tensor lr_sparse with slice shape (10,), updater FTRLTensorUpdater(1.0, 1.0, 0.02, 1.0) and initializer NormalTensorInitializer(0.0, 0.01)\u001b[m\n",
      "\u001b[38;5;196minit sparse tensor dnn_sparse with slice shape (10,), updater FTRLTensorUpdater(1.0, 1.0, 0.02, 1.0) and initializer NormalTensorInitializer(0.0, 0.01)\u001b[m\n",
      "\u001b[38;5;046minit dense tensor dnn.dnn.0.weight with shape (20, 1), updater AdamTensorUpdater(1e-05, 0.9, 0.999, 1e-08) and initializer OneTensorInitializer()\u001b[m\n",
      "\u001b[38;5;046minit dense tensor dnn.dnn.0.bias with shape (20, 1), updater AdamTensorUpdater(1e-05, 0.9, 0.999, 1e-08) and initializer ZeroTensorInitializer()\u001b[m\n",
      "\u001b[38;5;046minit dense tensor dnn.dnn.1.weight with shape (1024, 20), updater AdamTensorUpdater(1e-05, 0.9, 0.999, 1e-08) and initializer DefaultTensorInitializer()\u001b[m\n",
      "\u001b[38;5;046minit dense tensor dnn.dnn.1.bias with shape (1024, 1), updater AdamTensorUpdater(1e-05, 0.9, 0.999, 1e-08) and initializer DefaultTensorInitializer()\u001b[m\n",
      "\u001b[38;5;046minit dense tensor dnn.dnn.3.weight with shape (512, 1024), updater AdamTensorUpdater(1e-05, 0.9, 0.999, 1e-08) and initializer DefaultTensorInitializer()\u001b[m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2022-05-23 20:07:17.165] [info] C[0]:9: The coordinator has connected to 1 servers and 1 workers.\n",
      "PS Coordinator node \u001b[32mC[0]:9\u001b[m is ready.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[38;5;046minit dense tensor dnn.dnn.3.bias with shape (512, 1), updater AdamTensorUpdater(1e-05, 0.9, 0.999, 1e-08) and initializer DefaultTensorInitializer()\u001b[m\n",
      "\u001b[38;5;046minit dense tensor dnn.dnn.5.weight with shape (256, 512), updater AdamTensorUpdater(1e-05, 0.9, 0.999, 1e-08) and initializer DefaultTensorInitializer()\u001b[m\n",
      "\u001b[38;5;046minit dense tensor dnn.dnn.5.bias with shape (256, 1), updater AdamTensorUpdater(1e-05, 0.9, 0.999, 1e-08) and initializer DefaultTensorInitializer()\u001b[m\n",
      "\u001b[38;5;046minit dense tensor dnn.dnn.7.weight with shape (128, 256), updater AdamTensorUpdater(1e-05, 0.9, 0.999, 1e-08) and initializer DefaultTensorInitializer()\u001b[m\n",
      "\u001b[38;5;046minit dense tensor dnn.dnn.7.bias with shape (128, 1), updater AdamTensorUpdater(1e-05, 0.9, 0.999, 1e-08) and initializer DefaultTensorInitializer()\u001b[m\n",
      "\u001b[38;5;046minit dense tensor dnn.dnn.9.weight with shape (1, 128), updater AdamTensorUpdater(1e-05, 0.9, 0.999, 1e-08) and initializer DefaultTensorInitializer()\u001b[m\n",
      "\u001b[38;5;046minit dense tensor dnn.dnn.9.bias with shape (1, 1), updater AdamTensorUpdater(1e-05, 0.9, 0.999, 1e-08) and initializer DefaultTensorInitializer()\u001b[m\n",
      "\u001b[38;5;051minit dense tensor dnn.dnn.0.running_mean with shape (20, 1), updater EMATensorUpdater(0.1) and initializer ZeroTensorInitializer()\u001b[m\n",
      "\u001b[38;5;051minit dense tensor dnn.dnn.0.running_var with shape (20, 1), updater EMATensorUpdater(0.1) and initializer OneTensorInitializer()\u001b[m\n",
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:>               (3 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:07:46.403 -- auc: 0.49422377225527303, Δauc: 0.49422377225527303, pcoc: 0.7836304284398992, Δpcoc: 0.7836304284398992, #instance: 12533\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:>               (6 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:07:50.401 -- auc: 0.49890712418103467, Δauc: 0.497904490442715, pcoc: 0.8683975595478327, Δpcoc: 0.9515381753143699, #instance: 25272\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:>              (10 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:07:54.466 -- auc: 0.5287786246112418, Δauc: 0.642067587764524, pcoc: 0.90941921775345, Δpcoc: 0.9915295151353235, #instance: 37806\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:>              (13 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:07:58.466 -- auc: 0.5888355307627691, Δauc: 0.7513183425581421, pcoc: 0.9320910861776546, Δpcoc: 1.000450662390994, #instance: 50341\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:=>             (16 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:08:02.436 -- auc: 0.6396572668453981, Δauc: 0.7937302880246894, pcoc: 0.9470856641611272, Δpcoc: 1.0070823574348688, #instance: 62980\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:=>             (20 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:08:06.473 -- auc: 0.6742934204180482, Δauc: 0.8047745367908747, pcoc: 0.956052296589052, Δpcoc: 1.0007457830250033, #instance: 75580\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:=>             (23 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:08:10.416 -- auc: 0.7006104337414019, Δauc: 0.8205197035683711, pcoc: 0.9624447314459359, Δpcoc: 1.000958561750649, #instance: 88099\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:==>            (27 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:08:14.453 -- auc: 0.721036653930323, Δauc: 0.8314867025046908, pcoc: 0.9672436316047122, Δpcoc: 1.0008212347064718, #instance: 100555\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:==>            (30 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:08:18.407 -- auc: 0.7360609505618323, Δauc: 0.8332981718158742, pcoc: 0.9713106417069155, Δpcoc: 1.003957587936223, #instance: 113144\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:==>            (33 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:08:22.394 -- auc: 0.748514233165499, Δauc: 0.8391457025607515, pcoc: 0.9741423670836263, Δpcoc: 0.9993435213555709, #instance: 125799\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:==>            (37 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:08:26.389 -- auc: 0.7588248959939028, Δauc: 0.8437465538368657, pcoc: 0.9769481618805929, Δpcoc: 1.0054529424897742, #instance: 138336\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:===>           (40 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:08:30.445 -- auc: 0.7674185914844701, Δauc: 0.8453951452211194, pcoc: 0.9787015810761907, Δpcoc: 0.9977837106455927, #instance: 150987\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:===>           (44 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:08:34.449 -- auc: 0.7742558175839999, Δauc: 0.8442803128560836, pcoc: 0.9803677148358024, Δpcoc: 1.0003642125444767, #instance: 163538\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:===>           (47 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:08:38.401 -- auc: 0.7807961080018897, Δauc: 0.8512976513711672, pcoc: 0.9818010484564217, Δpcoc: 1.0002652802931011, #instance: 176230\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:===>           (51 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:08:42.414 -- auc: 0.7860887028092984, Δauc: 0.8498806257375501, pcoc: 0.9829759262143312, Δpcoc: 0.9996190370944708, #instance: 188749\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:====>          (54 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:08:46.358 -- auc: 0.79147000412859, Δauc: 0.8593051655420713, pcoc: 0.9840582408852915, Δpcoc: 1.0001892871750242, #instance: 201409\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:====>          (58 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:08:50.355 -- auc: 0.7960490818889361, Δauc: 0.8583267956617575, pcoc: 0.984971073708194, Δpcoc: 0.9994507584901441, #instance: 214131\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:====>          (61 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:08:54.310 -- auc: 0.7997948233523399, Δauc: 0.8554564624016072, pcoc: 0.9859496461635634, Δpcoc: 1.0025498097551206, #instance: 226760\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:====>          (65 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:08:58.323 -- auc: 0.8032455380859316, Δauc: 0.8571086709279416, pcoc: 0.9864773623756669, Δpcoc: 0.9960891844823386, #instance: 239243\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:=====>         (68 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:09:02.278 -- auc: 0.8060230381641149, Δauc: 0.851965315615598, pcoc: 0.9873046011333659, Δpcoc: 1.0030271214045288, #instance: 251847\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:=====>         (72 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:09:06.241 -- auc: 0.8089253229479643, Δauc: 0.8598191732923223, pcoc: 0.9877668545595187, Δpcoc: 0.9969752154134528, #instance: 264473\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:=====>         (75 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:09:10.206 -- auc: 0.8114624954279942, Δauc: 0.8597447048081837, pcoc: 0.9884736587118131, Δpcoc: 1.003316257359617, #instance: 277049\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:=====>         (78 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:09:14.195 -- auc: 0.8141558753859014, Δauc: 0.8662475028469393, pcoc: 0.9888086279058417, Δpcoc: 0.9961381659944848, #instance: 289743\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:======>        (82 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:09:18.185 -- auc: 0.8162431765845226, Δauc: 0.8600389005711304, pcoc: 0.9893339411507548, Δpcoc: 1.0016104762822373, #instance: 302194\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:======>        (85 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:09:22.149 -- auc: 0.8182342918518255, Δauc: 0.8613128029867511, pcoc: 0.9898772179238875, Δpcoc: 1.0029007894660915, #instance: 314763\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:======>        (89 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:09:26.140 -- auc: 0.820061304168748, Δauc: 0.8621031652971993, pcoc: 0.9904282302819968, Δpcoc: 1.0043947964227964, #instance: 327204\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:======>        (92 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:09:30.088 -- auc: 0.8218061586296854, Δauc: 0.8632396108616541, pcoc: 0.9907622314317446, Δpcoc: 0.9993186113638799, #instance: 339911\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:=======>       (96 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:09:34.102 -- auc: 0.8234311215090402, Δauc: 0.8638005214885462, pcoc: 0.991096399717256, Δpcoc: 1.0001451540452253, #instance: 352477\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:=======>       (99 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:09:38.041 -- auc: 0.8252315015181787, Δauc: 0.8706568910715318, pcoc: 0.9914616811757274, Δpcoc: 1.0017763027237907, #instance: 365035\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:=======>      (103 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:09:42.055 -- auc: 0.8265862361383689, Δauc: 0.8628463204855064, pcoc: 0.9917454213277063, Δpcoc: 0.9999692803161297, #instance: 377625\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:=======>      (106 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:09:46.058 -- auc: 0.8280680646117167, Δauc: 0.8683902532904393, pcoc: 0.9920244667240932, Δpcoc: 1.0003717009508144, #instance: 390164\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:=======>      (110 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:09:50.064 -- auc: 0.8291447116444933, Δauc: 0.8605978257977581, pcoc: 0.9922559001298171, Δpcoc: 0.9994491518166327, #instance: 402624\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:=======>      (113 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:09:54.070 -- auc: 0.8303347022431606, Δauc: 0.8659471315001728, pcoc: 0.9925274070550318, Δpcoc: 1.001115468509988, #instance: 415243\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:========>     (117 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:09:58.140 -- auc: 0.8317132478841627, Δauc: 0.8727909171877761, pcoc: 0.9928398010852153, Δpcoc: 1.0032019918579578, #instance: 427793\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:========>     (120 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:10:02.137 -- auc: 0.8329857937538236, Δauc: 0.8725854841723397, pcoc: 0.9931091411343312, Δpcoc: 1.0022777451680074, #instance: 440358\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:========>     (124 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:10:06.186 -- auc: 0.8341153830133963, Δauc: 0.871082122514588, pcoc: 0.9933546925519995, Δpcoc: 1.0019100763769402, #instance: 452962\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:========>     (127 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:10:10.145 -- auc: 0.8352178469658977, Δauc: 0.8717008928680813, pcoc: 0.9934653879152017, Δpcoc: 0.9974760955804275, #instance: 465545\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:=========>    (130 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:10:14.146 -- auc: 0.8363562082154956, Δauc: 0.8748263468465465, pcoc: 0.9936237784984041, Δpcoc: 0.9994308562009495, #instance: 478195\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:=========>    (134 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:10:18.175 -- auc: 0.8374285848081741, Δauc: 0.8746925927358653, pcoc: 0.9938281099291347, Δpcoc: 1.0015512030415454, #instance: 490788\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:=========>    (137 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:10:22.167 -- auc: 0.8384347649793672, Δauc: 0.8741079564088388, pcoc: 0.9939581412253754, Δpcoc: 0.9990240443659734, #instance: 503456\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:=========>    (141 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:10:26.140 -- auc: 0.8392615799894467, Δauc: 0.870214609862788, pcoc: 0.9941442723970149, Δpcoc: 1.0016284645123625, #instance: 515983\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:==========>   (144 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:10:30.087 -- auc: 0.8400273999548222, Δauc: 0.8694263778462346, pcoc: 0.9943016989878417, Δpcoc: 1.0007573305665807, #instance: 528516\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:==========>   (148 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:10:34.136 -- auc: 0.8409779967933007, Δauc: 0.8774578123609937, pcoc: 0.9943880892001025, Δpcoc: 0.9979920792973751, #instance: 541180\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:==========>   (151 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:10:38.082 -- auc: 0.8416458645896819, Δauc: 0.8685627418600791, pcoc: 0.9944302490645893, Δpcoc: 0.9962524671024203, #instance: 553730\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:==========>   (154 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:10:42.047 -- auc: 0.8424669988673367, Δauc: 0.8760784496668605, pcoc: 0.9946032368395659, Δpcoc: 1.0022303430503592, #instance: 566301\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:===========>  (158 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:10:46.105 -- auc: 0.8431529343766698, Δauc: 0.8719080079389613, pcoc: 0.9947184926681282, Δpcoc: 0.9999218158004934, #instance: 578849\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:===========>  (161 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:10:50.054 -- auc: 0.8438298259547852, Δauc: 0.8732212970792731, pcoc: 0.9947883596911961, Δpcoc: 0.9979668541844416, #instance: 591465\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:===========>  (164 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:10:54.055 -- auc: 0.8444170013517426, Δauc: 0.8701641096953727, pcoc: 0.994931524332662, Δpcoc: 1.001681122944879, #instance: 604073\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:===========>  (168 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:10:58.039 -- auc: 0.8449374801224437, Δauc: 0.8689805442878972, pcoc: 0.9950920884666914, Δpcoc: 1.0027847531032776, #instance: 616726\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:===========>  (171 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:11:02.005 -- auc: 0.8455860860085385, Δauc: 0.8752171997892438, pcoc: 0.9952085817049114, Δpcoc: 1.0009941312974435, #instance: 629205\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:============> (175 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:11:06.010 -- auc: 0.8461483619907917, Δauc: 0.8729239022356212, pcoc: 0.9950554580387787, Δpcoc: 0.9873406658178495, #instance: 641738\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:============> (178 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:11:09.937 -- auc: 0.8467937047464346, Δauc: 0.8775884323358968, pcoc: 0.9951825435744156, Δpcoc: 1.001597049189549, #instance: 654406\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:============> (182 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:11:13.936 -- auc: 0.8473807842706049, Δauc: 0.8764561942535799, pcoc: 0.9952860441191634, Δpcoc: 1.0006923074966823, #instance: 666910\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:============> (185 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:11:17.861 -- auc: 0.8479029661206665, Δauc: 0.8740445480143987, pcoc: 0.9953728788864882, Δpcoc: 0.9999203624692625, #instance: 679586\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:=============>(188 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:11:21.824 -- auc: 0.8484101718883047, Δauc: 0.8743503426642422, pcoc: 0.9954749602693321, Δpcoc: 1.0009456122416736, #instance: 692204\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:=============>(192 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:11:25.839 -- auc: 0.8488859261872121, Δauc: 0.87354896456235, pcoc: 0.9955914919338288, Δpcoc: 1.001945999804905, #instance: 704845\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:=============>(195 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:11:29.819 -- auc: 0.8493746646307279, Δauc: 0.8748765376148551, pcoc: 0.9956389968071322, Δpcoc: 0.998297133300831, #instance: 717448\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 63:>                 (0 + 1) / 1][Stage 79:=============>(198 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:11:33.800 -- auc: 0.8498796457504295, Δauc: 0.8771850143050635, pcoc: 0.9957329827907089, Δpcoc: 1.0011051315793287, #instance: 729994\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:11:35.484 -- auc: 0.8500847201867379, Δauc: 0.8817758831923865, pcoc: 0.9957564136858917, Δpcoc: 0.9996463119234915, #instance: 734372\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[38;5;196msaving model to s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/\u001b[m\n",
      "Get aws endpoint from env: obs.cn-southwest-2.myhuaweicloud.com\n",
      "[WARN] 2022-05-23 12:11:35.511 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:35.511] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/lr_sparse__sparse_meta.json, read_only false\n",
      "Get aws endpoint from env: obs.cn-southwest-2.myhuaweicloud.com\n",
      "[WARN] 2022-05-23 12:11:35.582 STSAssumeRoleWithWebIdentityCredentialsProvider [140353917589248] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:35.582] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/lr_sparse__sparse_0.dat, read_only false\n",
      "[WARN] 2022-05-23 12:11:35.640 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:35.640] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn_sparse__sparse_meta.json, read_only false\n",
      "[WARN] 2022-05-23 12:11:35.666 STSAssumeRoleWithWebIdentityCredentialsProvider [140353917589248] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:35.666] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn_sparse__sparse_0.dat, read_only false\n",
      "[WARN] 2022-05-23 12:11:35.728 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:35.728] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.weight__dense_meta.json, read_only false\n",
      "[WARN] 2022-05-23 12:11:35.760 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:35.760] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.weight__dense_data.dat, read_only false\n",
      "[WARN] 2022-05-23 12:11:35.786 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:35.786] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.weight__dense_state.dat, read_only false\n",
      "[WARN] 2022-05-23 12:11:35.812 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:35.812] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.bias__dense_meta.json, read_only false\n",
      "[WARN] 2022-05-23 12:11:35.838 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:35.838] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.bias__dense_data.dat, read_only false\n",
      "[WARN] 2022-05-23 12:11:35.869 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:35.869] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.bias__dense_state.dat, read_only false\n",
      "[WARN] 2022-05-23 12:11:35.895 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:35.895] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.1.weight__dense_meta.json, read_only false\n",
      "[WARN] 2022-05-23 12:11:35.924 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:35.924] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.1.weight__dense_data.dat, read_only false\n",
      "[WARN] 2022-05-23 12:11:35.952 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:35.952] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.1.weight__dense_state.dat, read_only false\n",
      "[WARN] 2022-05-23 12:11:35.982 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:35.982] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.1.bias__dense_meta.json, read_only false\n",
      "[WARN] 2022-05-23 12:11:36.007 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:36.007] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.1.bias__dense_data.dat, read_only false\n",
      "[WARN] 2022-05-23 12:11:36.034 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:36.034] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.1.bias__dense_state.dat, read_only false\n",
      "[WARN] 2022-05-23 12:11:36.061 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:36.061] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.3.weight__dense_meta.json, read_only false\n",
      "[WARN] 2022-05-23 12:11:36.088 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:36.088] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.3.weight__dense_data.dat, read_only false\n",
      "[WARN] 2022-05-23 12:11:36.148 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:36.148] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.3.weight__dense_state.dat, read_only false\n",
      "[WARN] 2022-05-23 12:11:36.232 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:36.232] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.3.bias__dense_meta.json, read_only false\n",
      "[WARN] 2022-05-23 12:11:36.259 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:36.259] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.3.bias__dense_data.dat, read_only false\n",
      "[WARN] 2022-05-23 12:11:36.286 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:36.286] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.3.bias__dense_state.dat, read_only false\n",
      "[WARN] 2022-05-23 12:11:36.315 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:36.315] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.5.weight__dense_meta.json, read_only false\n",
      "[WARN] 2022-05-23 12:11:36.341 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:36.341] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.5.weight__dense_data.dat, read_only false\n",
      "[WARN] 2022-05-23 12:11:36.375 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:36.375] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.5.weight__dense_state.dat, read_only false\n",
      "[WARN] 2022-05-23 12:11:36.420 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:36.420] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.5.bias__dense_meta.json, read_only false\n",
      "[WARN] 2022-05-23 12:11:36.449 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:36.449] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.5.bias__dense_data.dat, read_only false\n",
      "[WARN] 2022-05-23 12:11:36.475 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:36.475] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.5.bias__dense_state.dat, read_only false\n",
      "[WARN] 2022-05-23 12:11:36.502 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:36.502] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.7.weight__dense_meta.json, read_only false\n",
      "[WARN] 2022-05-23 12:11:36.528 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:36.528] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.7.weight__dense_data.dat, read_only false\n",
      "[WARN] 2022-05-23 12:11:36.559 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:36.559] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.7.weight__dense_state.dat, read_only false\n",
      "[WARN] 2022-05-23 12:11:36.590 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:36.590] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.7.bias__dense_meta.json, read_only false\n",
      "[WARN] 2022-05-23 12:11:36.615 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:36.615] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.7.bias__dense_data.dat, read_only false\n",
      "[WARN] 2022-05-23 12:11:36.647 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:36.647] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.7.bias__dense_state.dat, read_only false\n",
      "[WARN] 2022-05-23 12:11:36.682 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:36.682] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.9.weight__dense_meta.json, read_only false\n",
      "[WARN] 2022-05-23 12:11:36.709 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:36.709] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.9.weight__dense_data.dat, read_only false\n",
      "[WARN] 2022-05-23 12:11:36.737 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:36.737] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.9.weight__dense_state.dat, read_only false\n",
      "[WARN] 2022-05-23 12:11:36.764 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:36.764] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.9.bias__dense_meta.json, read_only false\n",
      "[WARN] 2022-05-23 12:11:36.790 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:36.790] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.9.bias__dense_data.dat, read_only false\n",
      "[WARN] 2022-05-23 12:11:36.816 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:36.816] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.9.bias__dense_state.dat, read_only false\n",
      "[WARN] 2022-05-23 12:11:36.846 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:36.846] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.running_mean__dense_meta.json, read_only false\n",
      "[WARN] 2022-05-23 12:11:36.873 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:36.873] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.running_mean__dense_data.dat, read_only false\n",
      "[WARN] 2022-05-23 12:11:36.899 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:36.899] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.running_var__dense_meta.json, read_only false\n",
      "[WARN] 2022-05-23 12:11:36.926 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:36.926] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.running_var__dense_data.dat, read_only false\n",
      "\u001b[38;5;196mexporting model to s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_export/\u001b[m\n",
      "/usr/local/lib/python3.8/dist-packages/torch/onnx/symbolic_helper.py:719: UserWarning: allowzero=0 by default. In order to honor zero value in shape use allowzero=1\n",
      "  warnings.warn(\"allowzero=0 by default. In order to honor zero value in shape use allowzero=1\")\n",
      "graph(%input : Long(*, strides=[1], requires_grad=0, device=cpu),\n",
      "      %weight : Float(*, 10, strides=[10, 1], requires_grad=0, device=cpu),\n",
      "      %offsets : Long(*, strides=[1], requires_grad=0, device=cpu),\n",
      "      %batch_size : Long(requires_grad=0, device=cpu),\n",
      "      %42 : Bool(requires_grad=0, device=cpu),\n",
      "      %43 : Long(1, strides=[1], requires_grad=0, device=cpu)):\n",
      "  %7 : Long(1, strides=[1], device=cpu) = onnx::Constant[value={0}]() # /usr/local/lib/python3.8/dist-packages/torch/nn/functional.py:2364:19\n",
      "  %8 : Long(1, strides=[1], device=cpu) = onnx::Shape(%input) # /usr/local/lib/python3.8/dist-packages/torch/nn/functional.py:2364:19\n",
      "  %9 : Long(device=cpu) = onnx::Constant[value={0}]() # /usr/local/lib/python3.8/dist-packages/torch/nn/functional.py:2364:19\n",
      "  %10 : Long(device=cpu) = onnx::Gather[axis=0](%8, %9) # /usr/local/lib/python3.8/dist-packages/torch/nn/functional.py:2364:19\n",
      "  %11 : Long(1, strides=[1], device=cpu) = onnx::Constant[value={0}]() # /usr/local/lib/python3.8/dist-packages/torch/nn/functional.py:2364:19\n",
      "  %12 : Long(1, strides=[1], device=cpu) = onnx::Unsqueeze(%10, %11) # /usr/local/lib/python3.8/dist-packages/torch/nn/functional.py:2364:19\n",
      "  %13 : Long(*, device=cpu) = onnx::Concat[axis=0](%offsets, %12) # /usr/local/lib/python3.8/dist-packages/torch/nn/functional.py:2364:19\n",
      "  %14 : Long(1, strides=[1], device=cpu) = onnx::Constant[value={0}]() # /usr/local/lib/python3.8/dist-packages/torch/nn/functional.py:2364:19\n",
      "  %15 : Long(1, strides=[1], device=cpu) = onnx::Constant[value={1}]() # /usr/local/lib/python3.8/dist-packages/torch/nn/functional.py:2364:19\n",
      "  %16 : Long(1, strides=[1], device=cpu) = onnx::Constant[value={9223372036854775807}]() # /usr/local/lib/python3.8/dist-packages/torch/nn/functional.py:2364:19\n",
      "  %17 : Long(1, strides=[1], device=cpu) = onnx::Constant[value={1}]() # /usr/local/lib/python3.8/dist-packages/torch/nn/functional.py:2364:19\n",
      "  %18 : Long(*, device=cpu) = onnx::Slice(%13, %15, %16, %14, %17) # /usr/local/lib/python3.8/dist-packages/torch/nn/functional.py:2364:19\n",
      "  %19 : Long(1, strides=[1], device=cpu) = onnx::Shape(%18) # /usr/local/lib/python3.8/dist-packages/torch/nn/functional.py:2364:19\n",
      "  %20 : Long(device=cpu) = onnx::Constant[value={0}]() # /usr/local/lib/python3.8/dist-packages/torch/nn/functional.py:2364:19\n",
      "  %21 : Long(device=cpu) = onnx::Gather[axis=0](%19, %20) # /usr/local/lib/python3.8/dist-packages/torch/nn/functional.py:2364:19\n",
      "  %embs : Float(*, *, device=cpu) = onnx::Loop(%21, %42) # /usr/local/lib/python3.8/dist-packages/torch/nn/functional.py:2364:19\n",
      "    block0(%23 : Long(device=cpu), %24 : Bool(requires_grad=0, device=cpu)):\n",
      "      %25 : Long(device=cpu) = onnx::Gather[axis=0](%13, %23)\n",
      "      %26 : Long(device=cpu) = onnx::Gather[axis=0](%18, %23)\n",
      "      %27 : Long(1, strides=[1], device=cpu) = onnx::Constant[value={0}]()\n",
      "      %28 : Long(*, strides=[1], device=cpu) = onnx::Unsqueeze(%25, %27)\n",
      "      %29 : Long(1, strides=[1], device=cpu) = onnx::Constant[value={0}]()\n",
      "      %30 : Long(*, strides=[1], device=cpu) = onnx::Unsqueeze(%26, %29)\n",
      "      %31 : Long(*, device=cpu) = onnx::Slice(%input, %28, %30, %7)\n",
      "      %32 : Float(*, *, device=cpu) = onnx::Gather[axis=0](%weight, %31)\n",
      "      %33 : Long(1, strides=[1], device=cpu) = onnx::Constant[value={0}]()\n",
      "      %34 : Float(*, device=cpu) = onnx::ReduceSum[keepdims=0, noop_with_empty_axes=0](%32, %33)\n",
      "      %35 : Bool(device=cpu) = onnx::Cast[to=9](%42)\n",
      "      -> (%35, %34)\n",
      "  %36 : Long(1, strides=[1], device=cpu) = onnx::Constant[value={0}]()\n",
      "  %37 : Long(1, strides=[1], device=cpu) = onnx::Unsqueeze(%batch_size, %36)\n",
      "  %40 : Long(2, strides=[1], device=cpu) = onnx::Concat[axis=0](%37, %43) # /usr/local/lib/python3.8/dist-packages/metaspore/embedding.py:42:15\n",
      "  %lr_sparse : Float(*, *, strides=[20, 1], requires_grad=0, device=cpu) = onnx::Reshape[allowzero=0](%embs, %40) # /usr/local/lib/python3.8/dist-packages/metaspore/embedding.py:42:15\n",
      "  return (%lr_sparse)\n",
      "\n",
      "[WARN] 2022-05-23 12:11:37.062 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:37.062] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_export/movielens_wdl_1m/sparse_lr_sparse/model.onnx, read_only false\n",
      "[WARN] 2022-05-23 12:11:37.083 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:37.084] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/wide_combine_column_schema, read_only true\n",
      "[2022-05-23 20:11:37.091] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/wide_combine_column_schema with total length 16\n",
      "[2022-05-23 20:11:37.094] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/wide_combine_column_schema with size 16 at position 0 larger than total size: 16, change size to 16\n",
      "[2022-05-23 20:11:37.099] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/wide_combine_column_schema reached end 16\n",
      "[2022-05-23 20:11:37.100] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/wide_combine_column_schema reached end 16\n",
      "[WARN] 2022-05-23 12:11:37.100 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:37.100] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_export/movielens_wdl_1m/sparse_lr_sparse/combine_schema.txt, read_only false\n",
      "[WARN] 2022-05-23 12:11:37.119 STSAssumeRoleWithWebIdentityCredentialsProvider [140353917589248] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:37.119] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_export/movielens_wdl_1m/sparse_lr_sparse/embedding_table/part_1_0.dat, read_only false\n",
      "graph(%input : Long(*, strides=[1], requires_grad=0, device=cpu),\n",
      "      %weight : Float(*, 10, strides=[10, 1], requires_grad=0, device=cpu),\n",
      "      %offsets : Long(*, strides=[1], requires_grad=0, device=cpu),\n",
      "      %batch_size : Long(requires_grad=0, device=cpu),\n",
      "      %42 : Bool(requires_grad=0, device=cpu),\n",
      "      %43 : Long(1, strides=[1], requires_grad=0, device=cpu)):\n",
      "  %7 : Long(1, strides=[1], device=cpu) = onnx::Constant[value={0}]() # /usr/local/lib/python3.8/dist-packages/torch/nn/functional.py:2364:19\n",
      "  %8 : Long(1, strides=[1], device=cpu) = onnx::Shape(%input) # /usr/local/lib/python3.8/dist-packages/torch/nn/functional.py:2364:19\n",
      "  %9 : Long(device=cpu) = onnx::Constant[value={0}]() # /usr/local/lib/python3.8/dist-packages/torch/nn/functional.py:2364:19\n",
      "  %10 : Long(device=cpu) = onnx::Gather[axis=0](%8, %9) # /usr/local/lib/python3.8/dist-packages/torch/nn/functional.py:2364:19\n",
      "  %11 : Long(1, strides=[1], device=cpu) = onnx::Constant[value={0}]() # /usr/local/lib/python3.8/dist-packages/torch/nn/functional.py:2364:19\n",
      "  %12 : Long(1, strides=[1], device=cpu) = onnx::Unsqueeze(%10, %11) # /usr/local/lib/python3.8/dist-packages/torch/nn/functional.py:2364:19\n",
      "  %13 : Long(*, device=cpu) = onnx::Concat[axis=0](%offsets, %12) # /usr/local/lib/python3.8/dist-packages/torch/nn/functional.py:2364:19\n",
      "  %14 : Long(1, strides=[1], device=cpu) = onnx::Constant[value={0}]() # /usr/local/lib/python3.8/dist-packages/torch/nn/functional.py:2364:19\n",
      "  %15 : Long(1, strides=[1], device=cpu) = onnx::Constant[value={1}]() # /usr/local/lib/python3.8/dist-packages/torch/nn/functional.py:2364:19\n",
      "  %16 : Long(1, strides=[1], device=cpu) = onnx::Constant[value={9223372036854775807}]() # /usr/local/lib/python3.8/dist-packages/torch/nn/functional.py:2364:19\n",
      "  %17 : Long(1, strides=[1], device=cpu) = onnx::Constant[value={1}]() # /usr/local/lib/python3.8/dist-packages/torch/nn/functional.py:2364:19\n",
      "  %18 : Long(*, device=cpu) = onnx::Slice(%13, %15, %16, %14, %17) # /usr/local/lib/python3.8/dist-packages/torch/nn/functional.py:2364:19\n",
      "  %19 : Long(1, strides=[1], device=cpu) = onnx::Shape(%18) # /usr/local/lib/python3.8/dist-packages/torch/nn/functional.py:2364:19\n",
      "  %20 : Long(device=cpu) = onnx::Constant[value={0}]() # /usr/local/lib/python3.8/dist-packages/torch/nn/functional.py:2364:19\n",
      "  %21 : Long(device=cpu) = onnx::Gather[axis=0](%19, %20) # /usr/local/lib/python3.8/dist-packages/torch/nn/functional.py:2364:19\n",
      "  %embs : Float(*, *, device=cpu) = onnx::Loop(%21, %42) # /usr/local/lib/python3.8/dist-packages/torch/nn/functional.py:2364:19\n",
      "    block0(%23 : Long(device=cpu), %24 : Bool(requires_grad=0, device=cpu)):\n",
      "      %25 : Long(device=cpu) = onnx::Gather[axis=0](%13, %23)\n",
      "      %26 : Long(device=cpu) = onnx::Gather[axis=0](%18, %23)\n",
      "      %27 : Long(1, strides=[1], device=cpu) = onnx::Constant[value={0}]()\n",
      "      %28 : Long(*, strides=[1], device=cpu) = onnx::Unsqueeze(%25, %27)\n",
      "      %29 : Long(1, strides=[1], device=cpu) = onnx::Constant[value={0}]()\n",
      "      %30 : Long(*, strides=[1], device=cpu) = onnx::Unsqueeze(%26, %29)\n",
      "      %31 : Long(*, device=cpu) = onnx::Slice(%input, %28, %30, %7)\n",
      "      %32 : Float(*, *, device=cpu) = onnx::Gather[axis=0](%weight, %31)\n",
      "      %33 : Long(1, strides=[1], device=cpu) = onnx::Constant[value={0}]()\n",
      "      %34 : Float(*, device=cpu) = onnx::ReduceSum[keepdims=0, noop_with_empty_axes=0](%32, %33)\n",
      "      %35 : Bool(device=cpu) = onnx::Cast[to=9](%42)\n",
      "      -> (%35, %34)\n",
      "  %36 : Long(1, strides=[1], device=cpu) = onnx::Constant[value={0}]()\n",
      "  %37 : Long(1, strides=[1], device=cpu) = onnx::Unsqueeze(%batch_size, %36)\n",
      "  %40 : Long(2, strides=[1], device=cpu) = onnx::Concat[axis=0](%37, %43) # /usr/local/lib/python3.8/dist-packages/metaspore/embedding.py:42:15\n",
      "  %dnn_sparse : Float(*, *, strides=[20, 1], requires_grad=0, device=cpu) = onnx::Reshape[allowzero=0](%embs, %40) # /usr/local/lib/python3.8/dist-packages/metaspore/embedding.py:42:15\n",
      "  return (%dnn_sparse)\n",
      "\n",
      "[WARN] 2022-05-23 12:11:37.172 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:37.172] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_export/movielens_wdl_1m/sparse_dnn_sparse/model.onnx, read_only false\n",
      "[WARN] 2022-05-23 12:11:37.190 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:37.190] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/deep_combine_column_schema, read_only true\n",
      "[2022-05-23 20:11:37.198] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/deep_combine_column_schema with total length 16\n",
      "[2022-05-23 20:11:37.201] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/deep_combine_column_schema with size 16 at position 0 larger than total size: 16, change size to 16\n",
      "[2022-05-23 20:11:37.207] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/deep_combine_column_schema reached end 16\n",
      "[2022-05-23 20:11:37.207] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/deep_combine_column_schema reached end 16\n",
      "[WARN] 2022-05-23 12:11:37.207 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:37.207] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_export/movielens_wdl_1m/sparse_dnn_sparse/combine_schema.txt, read_only false\n",
      "[WARN] 2022-05-23 12:11:37.226 STSAssumeRoleWithWebIdentityCredentialsProvider [140353917589248] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:37.226] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_export/movielens_wdl_1m/sparse_dnn_sparse/embedding_table/part_1_0.dat, read_only false\n",
      "name_list ['lr_sparse', 'dnn_sparse']\n",
      "node placeholder, x, x\n",
      "node call_module, lr_sparse, lr_sparse\n",
      "node call_function, sum_1, <built-in method sum of type object at 0x7fa6ce5e1780>\n",
      "node call_module, dnn_sparse, dnn_sparse\n",
      "node get_attr, dnn_dnn_0_weight, dnn.dnn.0.weight\n",
      "node get_attr, dnn_dnn_0_bias, dnn.dnn.0.bias\n",
      "node get_attr, dnn_dnn_0_running_mean, dnn.dnn.0.running_mean\n",
      "node get_attr, dnn_dnn_0_running_var, dnn.dnn.0.running_var\n",
      "node call_function, batch_norm, <function batch_norm at 0x7fa6b67f2820>\n",
      "node call_module, dnn_dnn_1, dnn.dnn.1\n",
      "node call_module, dnn_dnn_2, dnn.dnn.2\n",
      "node call_module, dnn_dnn_3, dnn.dnn.3\n",
      "node call_module, dnn_dnn_4, dnn.dnn.4\n",
      "node call_module, dnn_dnn_5, dnn.dnn.5\n",
      "node call_module, dnn_dnn_6, dnn.dnn.6\n",
      "node call_module, dnn_dnn_7, dnn.dnn.7\n",
      "node call_module, dnn_dnn_8, dnn.dnn.8\n",
      "node call_module, dnn_dnn_9, dnn.dnn.9\n",
      "node call_function, add, <built-in method add of type object at 0x7fa6ce5e1780>\n",
      "node call_module, final_activation, final_activation\n",
      "node output, output, output\n",
      "node placeholder, lr_sparse_1, lr_sparse_1\n",
      "node placeholder, dnn_sparse_1, dnn_sparse_1\n",
      "opcode         name                    target                                                  args                                                           kwargs\n",
      "-------------  ----------------------  ------------------------------------------------------  -------------------------------------------------------------  ------------------------------------------------------------------------------------------------------\n",
      "call_function  sum_1                   <built-in method sum of type object at 0x7fa6ce5e1780>  (lr_sparse_1,)                                                 {'dim': 1, 'keepdim': True}\n",
      "get_attr       dnn_dnn_0_weight        dnn.dnn.0.weight                                        ()                                                             {}\n",
      "get_attr       dnn_dnn_0_bias          dnn.dnn.0.bias                                          ()                                                             {}\n",
      "get_attr       dnn_dnn_0_running_mean  dnn.dnn.0.running_mean                                  ()                                                             {}\n",
      "get_attr       dnn_dnn_0_running_var   dnn.dnn.0.running_var                                   ()                                                             {}\n",
      "call_function  batch_norm              <function batch_norm at 0x7fa6b67f2820>                 (dnn_sparse_1, dnn_dnn_0_running_mean, dnn_dnn_0_running_var)  {'weight': dnn_dnn_0_weight, 'bias': dnn_dnn_0_bias, 'training': False, 'momentum': 0.1, 'eps': 1e-05}\n",
      "call_module    dnn_dnn_1               dnn.dnn.1                                               (batch_norm,)                                                  {}\n",
      "call_module    dnn_dnn_2               dnn.dnn.2                                               (dnn_dnn_1,)                                                   {}\n",
      "call_module    dnn_dnn_3               dnn.dnn.3                                               (dnn_dnn_2,)                                                   {}\n",
      "call_module    dnn_dnn_4               dnn.dnn.4                                               (dnn_dnn_3,)                                                   {}\n",
      "call_module    dnn_dnn_5               dnn.dnn.5                                               (dnn_dnn_4,)                                                   {}\n",
      "call_module    dnn_dnn_6               dnn.dnn.6                                               (dnn_dnn_5,)                                                   {}\n",
      "call_module    dnn_dnn_7               dnn.dnn.7                                               (dnn_dnn_6,)                                                   {}\n",
      "call_module    dnn_dnn_8               dnn.dnn.8                                               (dnn_dnn_7,)                                                   {}\n",
      "call_module    dnn_dnn_9               dnn.dnn.9                                               (dnn_dnn_8,)                                                   {}\n",
      "call_function  add                     <built-in method add of type object at 0x7fa6ce5e1780>  (sum_1, dnn_dnn_9)                                             {}\n",
      "call_module    final_activation        final_activation                                        (add,)                                                         {}\n",
      "output         output                  output                                                  (final_activation,)                                            {}\n",
      "placeholder    lr_sparse_1             lr_sparse_1                                             ()                                                             {}\n",
      "placeholder    dnn_sparse_1            dnn_sparse_1                                            ()                                                             {}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2022-05-23 20:11:37.428] [info] C[0]:9 has stopped.\n",
      "[2022-05-23 20:11:37.429] [info] PS job with coordinator address 172.16.0.198:39951 stopped.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "graph(%lr_sparse : Float(*, 20, strides=[20, 1], requires_grad=0, device=cpu),\n",
      "      %dnn_sparse : Float(*, 20, strides=[20, 1], requires_grad=0, device=cpu),\n",
      "      %dnn.dnn.0.weight : Float(20, strides=[1], requires_grad=0, device=cpu),\n",
      "      %dnn.dnn.0.bias : Float(20, strides=[1], requires_grad=0, device=cpu),\n",
      "      %dnn.dnn.0.running_mean : Float(20, strides=[1], requires_grad=0, device=cpu),\n",
      "      %dnn.dnn.0.running_var : Float(20, strides=[1], requires_grad=0, device=cpu),\n",
      "      %dnn.dnn.1.weight : Float(1024, 20, strides=[20, 1], requires_grad=0, device=cpu),\n",
      "      %dnn.dnn.1.bias : Float(1024, strides=[1], requires_grad=0, device=cpu),\n",
      "      %dnn.dnn.3.weight : Float(512, 1024, strides=[1024, 1], requires_grad=0, device=cpu),\n",
      "      %dnn.dnn.3.bias : Float(512, strides=[1], requires_grad=0, device=cpu),\n",
      "      %dnn.dnn.5.weight : Float(256, 512, strides=[512, 1], requires_grad=0, device=cpu),\n",
      "      %dnn.dnn.5.bias : Float(256, strides=[1], requires_grad=0, device=cpu),\n",
      "      %dnn.dnn.7.weight : Float(128, 256, strides=[256, 1], requires_grad=0, device=cpu),\n",
      "      %dnn.dnn.7.bias : Float(128, strides=[1], requires_grad=0, device=cpu),\n",
      "      %dnn.dnn.9.weight : Float(1, 128, strides=[128, 1], requires_grad=0, device=cpu),\n",
      "      %dnn.dnn.9.bias : Float(1, strides=[1], requires_grad=0, device=cpu)):\n",
      "  %16 : Long(1, strides=[1], device=cpu) = onnx::Constant[value={1}]()\n",
      "  %sum_1 : Float(*, 1, device=cpu) = onnx::ReduceSum[keepdims=1](%lr_sparse, %16) # <eval_with_key>.0:5:12\n",
      "  %batch_norm : Float(*, 20, device=cpu) = onnx::BatchNormalization[epsilon=1.0000000000000001e-05, momentum=0.90000000000000002, training_mode=0](%dnn_sparse, %dnn.dnn.0.weight, %dnn.dnn.0.bias, %dnn.dnn.0.running_mean, %dnn.dnn.0.running_var) # /usr/local/lib/python3.8/dist-packages/torch/nn/functional.py:2421:11\n",
      "  %dnn_dnn_1 : Float(*, 1024, device=cpu) = onnx::Gemm[alpha=1., beta=1., transB=1](%batch_norm, %dnn.dnn.1.weight, %dnn.dnn.1.bias) # /usr/local/lib/python3.8/dist-packages/torch/nn/modules/linear.py:103:15\n",
      "  %result : Float(*, 1024, device=cpu) = onnx::Relu(%dnn_dnn_1) # /usr/local/lib/python3.8/dist-packages/torch/nn/functional.py:1442:17\n",
      "  %dnn_dnn_3 : Float(*, 512, device=cpu) = onnx::Gemm[alpha=1., beta=1., transB=1](%result, %dnn.dnn.3.weight, %dnn.dnn.3.bias) # /usr/local/lib/python3.8/dist-packages/torch/nn/modules/linear.py:103:15\n",
      "  %result.3 : Float(*, 512, device=cpu) = onnx::Relu(%dnn_dnn_3) # /usr/local/lib/python3.8/dist-packages/torch/nn/functional.py:1442:17\n",
      "  %dnn_dnn_5 : Float(*, 256, device=cpu) = onnx::Gemm[alpha=1., beta=1., transB=1](%result.3, %dnn.dnn.5.weight, %dnn.dnn.5.bias) # /usr/local/lib/python3.8/dist-packages/torch/nn/modules/linear.py:103:15\n",
      "  %result.7 : Float(*, 256, device=cpu) = onnx::Relu(%dnn_dnn_5) # /usr/local/lib/python3.8/dist-packages/torch/nn/functional.py:1442:17\n",
      "  %dnn_dnn_7 : Float(*, 128, device=cpu) = onnx::Gemm[alpha=1., beta=1., transB=1](%result.7, %dnn.dnn.7.weight, %dnn.dnn.7.bias) # /usr/local/lib/python3.8/dist-packages/torch/nn/modules/linear.py:103:15\n",
      "  %result.11 : Float(*, 128, device=cpu) = onnx::Relu(%dnn_dnn_7) # /usr/local/lib/python3.8/dist-packages/torch/nn/functional.py:1442:17\n",
      "  %dnn_dnn_9 : Float(*, 1, device=cpu) = onnx::Gemm[alpha=1., beta=1., transB=1](%result.11, %dnn.dnn.9.weight, %dnn.dnn.9.bias) # /usr/local/lib/python3.8/dist-packages/torch/nn/modules/linear.py:103:15\n",
      "  %add : Float(*, 1, device=cpu) = onnx::Add(%sum_1, %dnn_dnn_9) # <eval_with_key>.0:20:10\n",
      "  %output : Float(*, 1, strides=[1, 1], requires_grad=1, device=cpu) = onnx::Sigmoid(%add) # /usr/local/lib/python3.8/dist-packages/torch/nn/modules/activation.py:293:15\n",
      "  return (%output)\n",
      "\n",
      "[WARN] 2022-05-23 12:11:37.357 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:37.357] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_export/movielens_wdl_1m/_dense/model.onnx, read_only false\n",
      "[2022-05-23 20:11:37.428] [info] S[0]:10 has stopped.                           \n",
      "[2022-05-23 20:11:37.428] [info] W[0]:12 has stopped.\n",
      "[2022-05-23 20:11:37.429] [info] PS job with coordinator address 172.16.0.198:39951 stopped.\n"
     ]
    }
   ],
   "source": [
    "model = train(spark, rank_train_dataset, **params)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "75d56233-da00-4284-9b39-4ca7e1db8b6e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-23T10:54:39.376348Z",
     "iopub.status.busy": "2022-05-23T10:54:39.375696Z",
     "iopub.status.idle": "2022-05-23T10:54:39.379374Z",
     "shell.execute_reply": "2022-05-23T10:54:39.378818Z",
     "shell.execute_reply.started": "2022-05-23T10:54:39.376275Z"
    }
   },
   "source": [
    "#### 2.3 模型验证\n",
    "我们打印出模型在测试集的 AUC。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "57746a35-ea20-43e5-865e-79836ad10b66",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-23T12:11:37.436614Z",
     "iopub.status.busy": "2022-05-23T12:11:37.436454Z",
     "iopub.status.idle": "2022-05-23T12:12:02.000545Z",
     "shell.execute_reply": "2022-05-23T12:12:01.999759Z",
     "shell.execute_reply.started": "2022-05-23T12:11:37.436594Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2022-05-23 20:11:37.476] [info] PS job with coordinator address 172.16.0.198:39951 stopped.\n",
      "\u001b[38;5;196mps agent deregistered for process 8383 thread 0x7fa6d4083740\u001b[m\n",
      "[2022-05-23 20:11:37.485] [info] PS job with coordinator address 172.16.0.198:44305 started.\n",
      "[2022-05-23 20:11:37.485] [info] PSRunner::RunPS: pid: 8383, tid: 8383, thread: 0x7fa6d4083740\n",
      "[2022-05-23 20:11:37.485] [info] PSRunner::RunPSServer: pid: 8383, tid: 8383, thread: 0x7fa6d4083740\n",
      "[2022-05-23 20:11:37.485] [info] ActorProcess::Receiving: Server pid: 8383, tid: 9033, thread: 0x7fa6b1669700\n",
      "[2022-05-23 20:11:37.487] [info] PS job with coordinator address 172.16.0.198:44305 started.\n",
      "[2022-05-23 20:11:37.487] [info] PSRunner::RunPS: pid: 8382, tid: 9037, thread: 0x7fa6b266b700\n",
      "[2022-05-23 20:11:37.487] [info] PSRunner::RunPSWorker: pid: 8382, tid: 9037, thread: 0x7fa6b266b700\n",
      "\u001b[38;5;046mps agent registered for process 8382 thread 0x7fa6d4083740\u001b[m\n",
      "[2022-05-23 20:11:37.488] [info] ActorProcess::Receiving: Worker pid: 8382, tid: 9040, thread: 0x7fa6a22fb700\n",
      "[2022-05-23 20:11:37.489] [info] W[0]:12 has connected to others.\n",
      "PS Worker node \u001b[38;5;051mW[0]:12\u001b[m is ready.\n",
      "[2022-05-23 20:11:37.489] [info] S[0]:10 has connected to others.\n",
      "PS Server node \u001b[38;5;196mS[0]:10\u001b[m is ready.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2022-05-23 20:11:37.464] [info] PS job with coordinator address 172.16.0.198:44305 started.\n",
      "[2022-05-23 20:11:37.464] [info] PSRunner::RunPS: pid: 7895, tid: 9024, thread: 0x7f56f2cc6700\n",
      "[2022-05-23 20:11:37.464] [info] PSRunner::RunPSCoordinator: pid: 7895, tid: 9024, thread: 0x7f56f2cc6700\n",
      "[2022-05-23 20:11:37.467] [info] ActorProcess::Receiving: Coordinator pid: 7895, tid: 9027, thread: 0x7f5721151700\n",
      "[2022-05-23 20:11:37.488] [info] C[0]:9: The coordinator has connected to 1 servers and 1 workers.\n",
      "PS Coordinator node \u001b[32mC[0]:9\u001b[m is ready.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[38;5;196minit sparse tensor lr_sparse with slice shape (10,), updater FTRLTensorUpdater(1.0, 1.0, 0.02, 1.0) and initializer NormalTensorInitializer(0.0, 0.01)\u001b[m\n",
      "\u001b[38;5;196minit sparse tensor dnn_sparse with slice shape (10,), updater FTRLTensorUpdater(1.0, 1.0, 0.02, 1.0) and initializer NormalTensorInitializer(0.0, 0.01)\u001b[m\n",
      "\u001b[38;5;046minit dense tensor dnn.dnn.0.weight with shape (20, 1), updater AdamTensorUpdater(1e-05, 0.9, 0.999, 1e-08) and initializer OneTensorInitializer()\u001b[m\n",
      "\u001b[38;5;046minit dense tensor dnn.dnn.0.bias with shape (20, 1), updater AdamTensorUpdater(1e-05, 0.9, 0.999, 1e-08) and initializer ZeroTensorInitializer()\u001b[m\n",
      "\u001b[38;5;046minit dense tensor dnn.dnn.1.weight with shape (1024, 20), updater AdamTensorUpdater(1e-05, 0.9, 0.999, 1e-08) and initializer DefaultTensorInitializer()\u001b[m\n",
      "\u001b[38;5;046minit dense tensor dnn.dnn.1.bias with shape (1024, 1), updater AdamTensorUpdater(1e-05, 0.9, 0.999, 1e-08) and initializer DefaultTensorInitializer()\u001b[m\n",
      "\u001b[38;5;046minit dense tensor dnn.dnn.3.weight with shape (512, 1024), updater AdamTensorUpdater(1e-05, 0.9, 0.999, 1e-08) and initializer DefaultTensorInitializer()\u001b[m\n",
      "\u001b[38;5;046minit dense tensor dnn.dnn.3.bias with shape (512, 1), updater AdamTensorUpdater(1e-05, 0.9, 0.999, 1e-08) and initializer DefaultTensorInitializer()\u001b[m\n",
      "\u001b[38;5;046minit dense tensor dnn.dnn.5.weight with shape (256, 512), updater AdamTensorUpdater(1e-05, 0.9, 0.999, 1e-08) and initializer DefaultTensorInitializer()\u001b[m\n",
      "\u001b[38;5;046minit dense tensor dnn.dnn.5.bias with shape (256, 1), updater AdamTensorUpdater(1e-05, 0.9, 0.999, 1e-08) and initializer DefaultTensorInitializer()\u001b[m\n",
      "\u001b[38;5;046minit dense tensor dnn.dnn.7.weight with shape (128, 256), updater AdamTensorUpdater(1e-05, 0.9, 0.999, 1e-08) and initializer DefaultTensorInitializer()\u001b[m\n",
      "\u001b[38;5;046minit dense tensor dnn.dnn.7.bias with shape (128, 1), updater AdamTensorUpdater(1e-05, 0.9, 0.999, 1e-08) and initializer DefaultTensorInitializer()\u001b[m\n",
      "\u001b[38;5;046minit dense tensor dnn.dnn.9.weight with shape (1, 128), updater AdamTensorUpdater(1e-05, 0.9, 0.999, 1e-08) and initializer DefaultTensorInitializer()\u001b[m\n",
      "\u001b[38;5;046minit dense tensor dnn.dnn.9.bias with shape (1, 1), updater AdamTensorUpdater(1e-05, 0.9, 0.999, 1e-08) and initializer DefaultTensorInitializer()\u001b[m\n",
      "\u001b[38;5;051minit dense tensor dnn.dnn.0.running_mean with shape (20, 1), updater EMATensorUpdater(0.1) and initializer ZeroTensorInitializer()\u001b[m\n",
      "\u001b[38;5;051minit dense tensor dnn.dnn.0.running_var with shape (20, 1), updater EMATensorUpdater(0.1) and initializer OneTensorInitializer()\u001b[m\n",
      "\u001b[38;5;196mloading model from s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/\u001b[m\n",
      "[WARN] 2022-05-23 12:11:37.742 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:37.743] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/lr_sparse__sparse_meta.json, read_only true\n",
      "[2022-05-23 20:11:37.750] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/lr_sparse__sparse_meta.json with total length 508\n",
      "[2022-05-23 20:11:37.753] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/lr_sparse__sparse_meta.json with size 508 at position 0 larger than total size: 508, change size to 508\n",
      "[2022-05-23 20:11:37.758] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/lr_sparse__sparse_meta.json reached end 508\n",
      "[2022-05-23 20:11:37.759] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/lr_sparse__sparse_meta.json reached end 508\n",
      "[WARN] 2022-05-23 12:11:37.760 STSAssumeRoleWithWebIdentityCredentialsProvider [140353917589248] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:37.760] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/lr_sparse__sparse_0.dat, read_only true\n",
      "[2022-05-23 20:11:37.766] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/lr_sparse__sparse_0.dat with total length 1385648\n",
      "[2022-05-23 20:11:37.769] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/lr_sparse__sparse_0.dat with size 1385648 at position 0 larger than total size: 1385648, change size to 1385648\n",
      "[WARN] 2022-05-23 12:11:37.779 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:37.779] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn_sparse__sparse_meta.json, read_only true\n",
      "[2022-05-23 20:11:37.785] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn_sparse__sparse_meta.json with total length 509\n",
      "[2022-05-23 20:11:37.788] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn_sparse__sparse_meta.json with size 509 at position 0 larger than total size: 509, change size to 509\n",
      "[2022-05-23 20:11:37.792] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn_sparse__sparse_meta.json reached end 509\n",
      "[2022-05-23 20:11:37.792] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn_sparse__sparse_meta.json reached end 509\n",
      "[WARN] 2022-05-23 12:11:37.792 STSAssumeRoleWithWebIdentityCredentialsProvider [140353917589248] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:37.792] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn_sparse__sparse_0.dat, read_only true\n",
      "[2022-05-23 20:11:37.799] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn_sparse__sparse_0.dat with total length 1385648\n",
      "[2022-05-23 20:11:37.800] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn_sparse__sparse_0.dat with size 1385648 at position 0 larger than total size: 1385648, change size to 1385648\n",
      "[WARN] 2022-05-23 12:11:37.810 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:37.810] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.weight__dense_meta.json, read_only true\n",
      "[2022-05-23 20:11:37.816] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.weight__dense_meta.json with total length 443\n",
      "[2022-05-23 20:11:37.818] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.weight__dense_meta.json with size 443 at position 0 larger than total size: 443, change size to 443\n",
      "[2022-05-23 20:11:37.822] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.weight__dense_meta.json reached end 443\n",
      "[2022-05-23 20:11:37.823] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.weight__dense_meta.json reached end 443\n",
      "[WARN] 2022-05-23 12:11:37.823 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:37.823] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.weight__dense_data.dat, read_only true\n",
      "[2022-05-23 20:11:37.831] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.weight__dense_data.dat with total length 80\n",
      "[2022-05-23 20:11:37.831] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.weight__dense_data.dat with size 80 at position 0 larger than total size: 80, change size to 80\n",
      "[WARN] 2022-05-23 12:11:37.836 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:37.836] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.weight__dense_state.dat, read_only true\n",
      "[2022-05-23 20:11:37.843] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.weight__dense_state.dat with total length 160\n",
      "[2022-05-23 20:11:37.844] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.weight__dense_state.dat with size 160 at position 0 larger than total size: 160, change size to 160\n",
      "[WARN] 2022-05-23 12:11:37.849 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:37.849] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.bias__dense_meta.json, read_only true\n",
      "[2022-05-23 20:11:37.855] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.bias__dense_meta.json with total length 441\n",
      "[2022-05-23 20:11:37.857] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.bias__dense_meta.json with size 441 at position 0 larger than total size: 441, change size to 441\n",
      "[2022-05-23 20:11:37.861] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.bias__dense_meta.json reached end 441\n",
      "[2022-05-23 20:11:37.861] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.bias__dense_meta.json reached end 441\n",
      "[WARN] 2022-05-23 12:11:37.862 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:37.862] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.bias__dense_data.dat, read_only true\n",
      "[2022-05-23 20:11:37.868] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.bias__dense_data.dat with total length 80\n",
      "[2022-05-23 20:11:37.869] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.bias__dense_data.dat with size 80 at position 0 larger than total size: 80, change size to 80\n",
      "[WARN] 2022-05-23 12:11:37.874 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:37.874] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.bias__dense_state.dat, read_only true\n",
      "[2022-05-23 20:11:37.881] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.bias__dense_state.dat with total length 160\n",
      "[2022-05-23 20:11:37.882] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.bias__dense_state.dat with size 160 at position 0 larger than total size: 160, change size to 160\n",
      "[WARN] 2022-05-23 12:11:37.888 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:37.888] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.1.weight__dense_meta.json, read_only true\n",
      "[2022-05-23 20:11:37.894] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.1.weight__dense_meta.json with total length 453\n",
      "[2022-05-23 20:11:37.896] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.1.weight__dense_meta.json with size 453 at position 0 larger than total size: 453, change size to 453\n",
      "[2022-05-23 20:11:37.900] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.1.weight__dense_meta.json reached end 453\n",
      "[2022-05-23 20:11:37.900] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.1.weight__dense_meta.json reached end 453\n",
      "[WARN] 2022-05-23 12:11:37.900 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:37.900] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.1.weight__dense_data.dat, read_only true\n",
      "[2022-05-23 20:11:37.907] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.1.weight__dense_data.dat with total length 81920\n",
      "[2022-05-23 20:11:37.908] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.1.weight__dense_data.dat with size 81920 at position 0 larger than total size: 81920, change size to 81920\n",
      "[WARN] 2022-05-23 12:11:37.914 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:37.914] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.1.weight__dense_state.dat, read_only true\n",
      "[2022-05-23 20:11:37.920] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.1.weight__dense_state.dat with total length 163840\n",
      "[2022-05-23 20:11:37.921] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.1.weight__dense_state.dat with size 163840 at position 0 larger than total size: 163840, change size to 163840\n",
      "[WARN] 2022-05-23 12:11:37.926 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:37.926] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.1.bias__dense_meta.json, read_only true\n",
      "[2022-05-23 20:11:37.933] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.1.bias__dense_meta.json with total length 449\n",
      "[2022-05-23 20:11:37.934] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.1.bias__dense_meta.json with size 449 at position 0 larger than total size: 449, change size to 449\n",
      "[2022-05-23 20:11:37.938] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.1.bias__dense_meta.json reached end 449\n",
      "[2022-05-23 20:11:37.939] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.1.bias__dense_meta.json reached end 449\n",
      "[WARN] 2022-05-23 12:11:37.939 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:37.939] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.1.bias__dense_data.dat, read_only true\n",
      "[2022-05-23 20:11:37.945] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.1.bias__dense_data.dat with total length 4096\n",
      "[2022-05-23 20:11:37.946] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.1.bias__dense_data.dat with size 4096 at position 0 larger than total size: 4096, change size to 4096\n",
      "[WARN] 2022-05-23 12:11:37.950 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:37.950] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.1.bias__dense_state.dat, read_only true\n",
      "[2022-05-23 20:11:37.957] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.1.bias__dense_state.dat with total length 8192\n",
      "[2022-05-23 20:11:37.958] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.1.bias__dense_state.dat with size 8192 at position 0 larger than total size: 8192, change size to 8192\n",
      "[WARN] 2022-05-23 12:11:37.961 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:37.962] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.3.weight__dense_meta.json, read_only true\n",
      "[2022-05-23 20:11:37.968] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.3.weight__dense_meta.json with total length 455\n",
      "[2022-05-23 20:11:37.969] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.3.weight__dense_meta.json with size 455 at position 0 larger than total size: 455, change size to 455\n",
      "[2022-05-23 20:11:37.973] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.3.weight__dense_meta.json reached end 455\n",
      "[2022-05-23 20:11:37.973] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.3.weight__dense_meta.json reached end 455\n",
      "[WARN] 2022-05-23 12:11:37.974 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:37.974] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.3.weight__dense_data.dat, read_only true\n",
      "[2022-05-23 20:11:37.980] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.3.weight__dense_data.dat with total length 2097152\n",
      "[2022-05-23 20:11:37.981] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.3.weight__dense_data.dat with size 2097152 at position 0 larger than total size: 2097152, change size to 2097152\n",
      "[WARN] 2022-05-23 12:11:37.992 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:37.992] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.3.weight__dense_state.dat, read_only true\n",
      "[2022-05-23 20:11:37.999] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.3.weight__dense_state.dat with total length 4194304\n",
      "[2022-05-23 20:11:38.000] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.3.weight__dense_state.dat with size 4194304 at position 0 larger than total size: 4194304, change size to 4194304\n",
      "[WARN] 2022-05-23 12:11:38.017 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:38.017] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.3.bias__dense_meta.json, read_only true\n",
      "[2022-05-23 20:11:38.024] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.3.bias__dense_meta.json with total length 447\n",
      "[2022-05-23 20:11:38.025] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.3.bias__dense_meta.json with size 447 at position 0 larger than total size: 447, change size to 447\n",
      "[2022-05-23 20:11:38.029] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.3.bias__dense_meta.json reached end 447\n",
      "[2022-05-23 20:11:38.029] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.3.bias__dense_meta.json reached end 447\n",
      "[WARN] 2022-05-23 12:11:38.030 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:38.030] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.3.bias__dense_data.dat, read_only true\n",
      "[2022-05-23 20:11:38.036] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.3.bias__dense_data.dat with total length 2048\n",
      "[2022-05-23 20:11:38.036] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.3.bias__dense_data.dat with size 2048 at position 0 larger than total size: 2048, change size to 2048\n",
      "[WARN] 2022-05-23 12:11:38.040 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:38.040] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.3.bias__dense_state.dat, read_only true\n",
      "[2022-05-23 20:11:38.046] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.3.bias__dense_state.dat with total length 4096\n",
      "[2022-05-23 20:11:38.047] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.3.bias__dense_state.dat with size 4096 at position 0 larger than total size: 4096, change size to 4096\n",
      "[WARN] 2022-05-23 12:11:38.051 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:38.051] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.5.weight__dense_meta.json, read_only true\n",
      "[2022-05-23 20:11:38.057] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.5.weight__dense_meta.json with total length 454\n",
      "[2022-05-23 20:11:38.058] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.5.weight__dense_meta.json with size 454 at position 0 larger than total size: 454, change size to 454\n",
      "[2022-05-23 20:11:38.062] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.5.weight__dense_meta.json reached end 454\n",
      "[2022-05-23 20:11:38.063] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.5.weight__dense_meta.json reached end 454\n",
      "[WARN] 2022-05-23 12:11:38.063 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:38.063] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.5.weight__dense_data.dat, read_only true\n",
      "[2022-05-23 20:11:38.069] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.5.weight__dense_data.dat with total length 524288\n",
      "[2022-05-23 20:11:38.070] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.5.weight__dense_data.dat with size 524288 at position 0 larger than total size: 524288, change size to 524288\n",
      "[WARN] 2022-05-23 12:11:38.076 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:38.076] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.5.weight__dense_state.dat, read_only true\n",
      "[2022-05-23 20:11:38.082] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.5.weight__dense_state.dat with total length 1048576\n",
      "[2022-05-23 20:11:38.083] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.5.weight__dense_state.dat with size 1048576 at position 0 larger than total size: 1048576, change size to 1048576\n",
      "[WARN] 2022-05-23 12:11:38.091 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:38.091] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.5.bias__dense_meta.json, read_only true\n",
      "[2022-05-23 20:11:38.097] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.5.bias__dense_meta.json with total length 447\n",
      "[2022-05-23 20:11:38.098] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.5.bias__dense_meta.json with size 447 at position 0 larger than total size: 447, change size to 447\n",
      "[2022-05-23 20:11:38.102] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.5.bias__dense_meta.json reached end 447\n",
      "[2022-05-23 20:11:38.102] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.5.bias__dense_meta.json reached end 447\n",
      "[WARN] 2022-05-23 12:11:38.103 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:38.103] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.5.bias__dense_data.dat, read_only true\n",
      "[2022-05-23 20:11:38.109] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.5.bias__dense_data.dat with total length 1024\n",
      "[2022-05-23 20:11:38.109] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.5.bias__dense_data.dat with size 1024 at position 0 larger than total size: 1024, change size to 1024\n",
      "[WARN] 2022-05-23 12:11:38.114 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:38.114] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.5.bias__dense_state.dat, read_only true\n",
      "[2022-05-23 20:11:38.120] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.5.bias__dense_state.dat with total length 2048\n",
      "[2022-05-23 20:11:38.120] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.5.bias__dense_state.dat with size 2048 at position 0 larger than total size: 2048, change size to 2048\n",
      "[WARN] 2022-05-23 12:11:38.124 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:38.124] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.7.weight__dense_meta.json, read_only true\n",
      "[2022-05-23 20:11:38.130] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.7.weight__dense_meta.json with total length 453\n",
      "[2022-05-23 20:11:38.132] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.7.weight__dense_meta.json with size 453 at position 0 larger than total size: 453, change size to 453\n",
      "[2022-05-23 20:11:38.136] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.7.weight__dense_meta.json reached end 453\n",
      "[2022-05-23 20:11:38.136] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.7.weight__dense_meta.json reached end 453\n",
      "[WARN] 2022-05-23 12:11:38.136 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:38.136] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.7.weight__dense_data.dat, read_only true\n",
      "[2022-05-23 20:11:38.142] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.7.weight__dense_data.dat with total length 131072\n",
      "[2022-05-23 20:11:38.143] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.7.weight__dense_data.dat with size 131072 at position 0 larger than total size: 131072, change size to 131072\n",
      "[WARN] 2022-05-23 12:11:38.148 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:38.148] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.7.weight__dense_state.dat, read_only true\n",
      "[2022-05-23 20:11:38.154] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.7.weight__dense_state.dat with total length 262144\n",
      "[2022-05-23 20:11:38.154] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.7.weight__dense_state.dat with size 262144 at position 0 larger than total size: 262144, change size to 262144\n",
      "[WARN] 2022-05-23 12:11:38.168 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:38.168] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.7.bias__dense_meta.json, read_only true\n",
      "[2022-05-23 20:11:38.174] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.7.bias__dense_meta.json with total length 447\n",
      "[2022-05-23 20:11:38.175] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.7.bias__dense_meta.json with size 447 at position 0 larger than total size: 447, change size to 447\n",
      "[2022-05-23 20:11:38.179] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.7.bias__dense_meta.json reached end 447\n",
      "[2022-05-23 20:11:38.180] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.7.bias__dense_meta.json reached end 447\n",
      "[WARN] 2022-05-23 12:11:38.180 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:38.180] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.7.bias__dense_data.dat, read_only true\n",
      "[2022-05-23 20:11:38.187] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.7.bias__dense_data.dat with total length 512\n",
      "[2022-05-23 20:11:38.187] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.7.bias__dense_data.dat with size 512 at position 0 larger than total size: 512, change size to 512\n",
      "[WARN] 2022-05-23 12:11:38.191 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:38.191] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.7.bias__dense_state.dat, read_only true\n",
      "[2022-05-23 20:11:38.197] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.7.bias__dense_state.dat with total length 1024\n",
      "[2022-05-23 20:11:38.198] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.7.bias__dense_state.dat with size 1024 at position 0 larger than total size: 1024, change size to 1024\n",
      "[WARN] 2022-05-23 12:11:38.202 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:38.202] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.9.weight__dense_meta.json, read_only true\n",
      "[2022-05-23 20:11:38.208] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.9.weight__dense_meta.json with total length 449\n",
      "[2022-05-23 20:11:38.209] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.9.weight__dense_meta.json with size 449 at position 0 larger than total size: 449, change size to 449\n",
      "[2022-05-23 20:11:38.213] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.9.weight__dense_meta.json reached end 449\n",
      "[2022-05-23 20:11:38.214] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.9.weight__dense_meta.json reached end 449\n",
      "[WARN] 2022-05-23 12:11:38.214 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:38.214] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.9.weight__dense_data.dat, read_only true\n",
      "[2022-05-23 20:11:38.220] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.9.weight__dense_data.dat with total length 512\n",
      "[2022-05-23 20:11:38.220] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.9.weight__dense_data.dat with size 512 at position 0 larger than total size: 512, change size to 512\n",
      "[WARN] 2022-05-23 12:11:38.224 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:38.224] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.9.weight__dense_state.dat, read_only true\n",
      "[2022-05-23 20:11:38.231] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.9.weight__dense_state.dat with total length 1024\n",
      "[2022-05-23 20:11:38.232] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.9.weight__dense_state.dat with size 1024 at position 0 larger than total size: 1024, change size to 1024\n",
      "[WARN] 2022-05-23 12:11:38.236 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:38.236] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.9.bias__dense_meta.json, read_only true\n",
      "[2022-05-23 20:11:38.243] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.9.bias__dense_meta.json with total length 443\n",
      "[2022-05-23 20:11:38.244] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.9.bias__dense_meta.json with size 443 at position 0 larger than total size: 443, change size to 443\n",
      "[2022-05-23 20:11:38.249] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.9.bias__dense_meta.json reached end 443\n",
      "[2022-05-23 20:11:38.249] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.9.bias__dense_meta.json reached end 443\n",
      "[WARN] 2022-05-23 12:11:38.249 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:38.249] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.9.bias__dense_data.dat, read_only true\n",
      "[2022-05-23 20:11:38.257] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.9.bias__dense_data.dat with total length 4\n",
      "[2022-05-23 20:11:38.258] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.9.bias__dense_data.dat with size 4 at position 0 larger than total size: 4, change size to 4\n",
      "[WARN] 2022-05-23 12:11:38.262 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:38.262] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.9.bias__dense_state.dat, read_only true\n",
      "[2022-05-23 20:11:38.268] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.9.bias__dense_state.dat with total length 8\n",
      "[2022-05-23 20:11:38.269] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.9.bias__dense_state.dat with size 8 at position 0 larger than total size: 8, change size to 8\n",
      "[WARN] 2022-05-23 12:11:38.273 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:38.273] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.running_mean__dense_meta.json, read_only true\n",
      "[2022-05-23 20:11:38.280] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.running_mean__dense_meta.json with total length 369\n",
      "[2022-05-23 20:11:38.281] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.running_mean__dense_meta.json with size 369 at position 0 larger than total size: 369, change size to 369\n",
      "[2022-05-23 20:11:38.286] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.running_mean__dense_meta.json reached end 369\n",
      "[2022-05-23 20:11:38.286] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.running_mean__dense_meta.json reached end 369\n",
      "[WARN] 2022-05-23 12:11:38.287 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:38.287] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.running_mean__dense_data.dat, read_only true\n",
      "[2022-05-23 20:11:38.293] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.running_mean__dense_data.dat with total length 80\n",
      "[2022-05-23 20:11:38.294] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.running_mean__dense_data.dat with size 80 at position 0 larger than total size: 80, change size to 80\n",
      "[WARN] 2022-05-23 12:11:38.298 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:38.298] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.running_var__dense_meta.json, read_only true\n",
      "[2022-05-23 20:11:38.305] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.running_var__dense_meta.json with total length 368\n",
      "[2022-05-23 20:11:38.306] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.running_var__dense_meta.json with size 368 at position 0 larger than total size: 368, change size to 368\n",
      "[2022-05-23 20:11:38.310] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.running_var__dense_meta.json reached end 368\n",
      "[2022-05-23 20:11:38.310] [info] [s3_sdk_filesys.cpp:413] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.running_var__dense_meta.json reached end 368\n",
      "[WARN] 2022-05-23 12:11:38.310 STSAssumeRoleWithWebIdentityCredentialsProvider [140354498606912] Token file must be specified to use STS AssumeRole web identity creds provider.\n",
      "[2022-05-23 20:11:38.310] [info] [s3_sdk_filesys.cpp:357] Try to open S3 stream: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.running_var__dense_data.dat, read_only true\n",
      "[2022-05-23 20:11:38.316] [info] [s3_sdk_filesys.cpp:380] Opened read-only stream for object: s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.running_var__dense_data.dat with total length 80\n",
      "[2022-05-23 20:11:38.317] [info] [s3_sdk_filesys.cpp:419] Read S3 object s3://alphaide-demo/movielens/ml-1m/schema/widedeep/model_out/dnn.dnn.0.running_var__dense_data.dat with size 80 at position 0 larger than total size: 80, change size to 80\n",
      "[Stage 82:>                 (0 + 1) / 1][Stage 97:=======>       (99 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:11:53.751 -- auc: 0.8590159808331953, Δauc: 0.8590159808331953, pcoc: 0.9939411351199142, Δpcoc: 0.9939411351199142, #instance: 2321\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 82:>                 (0 + 1) / 1][Stage 97:=============>(196 + 1) / 200]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-23 20:11:59.074 -- auc: 0.8564639950217534, Δauc: 0.8539929548498687, pcoc: 0.9888738322643614, Δpcoc: 0.9837838695860895, #instance: 4640\n",
      "2022-05-23 20:11:59.156 -- auc: 0.8564639950217534, Δauc: 1.0, pcoc: 0.9888738322643614, Δpcoc: nan, #instance: 4640\n",
      "[2022-05-23 20:11:59.165] [info] C[0]:9 has stopped.\n",
      "[2022-05-23 20:11:59.166] [info] PS job with coordinator address 172.16.0.198:44305 stopped.\n",
      "Debug -- test result sample:\n",
      "+-----+-------+------+---+----------+-----+--------+--------------------+--------------------+--------------------+\n",
      "|label|user_id|gender|age|occupation|  zip|movie_id|               title|               genre|       rawPrediction|\n",
      "+-----+-------+------+---+----------+-----+--------+--------------------+--------------------+--------------------+\n",
      "|  1.0|   5966|     F| 35|         9|10021|     497|Much Ado About No...|      Comedy|Romance|  0.7502685785293579|\n",
      "|  0.0|   4173|     F| 18|        11|29063|      95| Broken Arrow (1996)|     Action|Thriller|  0.5468894839286804|\n",
      "|  1.0|   1718|     M| 18|         4|43235|    1396|     Sneakers (1992)|  Crime|Drama|Sci-Fi|  0.9273433685302734|\n",
      "|  0.0|    962|     F| 25|         1|80020|    3896|Way of the Gun, T...|      Crime|Thriller|  0.7319202423095703|\n",
      "|  1.0|    937|     M| 25|        15|60513|     750|Dr. Strangelove o...|          Sci-Fi|War|  0.9713308811187744|\n",
      "|  1.0|   3212|     F| 25|        17|11215|    2959|   Fight Club (1999)|               Drama|  0.7158253788948059|\n",
      "|  1.0|   4368|     M| 18|        17|22043|    1645|Devil's Advocate,...|Crime|Horror|Myst...|  0.7065671682357788|\n",
      "|  1.0|   2678|     M| 25|         6|49707|    2396|Shakespeare in Lo...|      Comedy|Romance|  0.9364445209503174|\n",
      "|  1.0|     63|     M| 18|         4|54902|     541| Blade Runner (1982)|    Film-Noir|Sci-Fi|  0.9315231442451477|\n",
      "|  1.0|    432|     M| 45|        16|55306|    3004|Bachelor, The (1999)|      Comedy|Romance|0.053500592708587646|\n",
      "|  1.0|   3117|     M| 35|         0|83704|    1200|       Aliens (1986)|Action|Sci-Fi|Thr...|  0.9287647604942322|\n",
      "|  1.0|   1763|     M| 35|         7|76248|    1257|Better Off Dead.....|              Comedy|  0.9172020554542542|\n",
      "|  0.0|     77|     M| 18|         4|15321|    2702|Summer of Sam (1999)|               Drama|  0.4198340177536011|\n",
      "|  1.0|    631|     M| 35|        17|28601|    1339|Bram Stoker's Dra...|      Horror|Romance|  0.7030164003372192|\n",
      "|  1.0|    789|     M| 18|         1|67212|    1259|  Stand by Me (1986)|Adventure|Comedy|...|  0.9460964798927307|\n",
      "|  1.0|   4056|     M|  1|        10|28270|    3000|Princess Mononoke...|Action|Adventure|...|  0.7306034564971924|\n",
      "|  1.0|   4661|     M| 45|        15|85255|    2858|American Beauty (...|        Comedy|Drama|   0.989676296710968|\n",
      "|  0.0|   2895|     M| 25|        11|07039|    1472|City of Industry ...|      Crime|Thriller|  0.6507465243339539|\n",
      "|  1.0|   1867|     M| 50|         7|95008|    1206|Clockwork Orange,...|              Sci-Fi|  0.6367302536964417|\n",
      "|  1.0|   2224|     M| 25|        12|60025|     720|Wallace & Gromit:...|           Animation|  0.9793381690979004|\n",
      "+-----+-------+------+---+----------+-----+--------+--------------------+--------------------+--------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[2022-05-23 20:11:59.165] [info] W[0]:12 has stopped.                           \n",
      "[2022-05-23 20:11:59.165] [info] S[0]:10 has stopped.\n",
      "[2022-05-23 20:11:59.166] [info] PS job with coordinator address 172.16.0.198:44305 stopped.\n",
      "[2022-05-23 20:11:59.223] [info] PS job with coordinator address 172.16.0.198:44305 stopped.\n",
      "\u001b[38;5;196mps agent deregistered for process 8382 thread 0x7fa6d4083740\u001b[m\n",
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Debug -- Test AUC:  0.8564576452173804\n"
     ]
    }
   ],
   "source": [
    " ## transform\n",
    "test_result = transform(spark, model, rank_test_dataset)\n",
    "## evaluate\n",
    "test_auc = evaluate(spark, test_result)\n",
    "print('Debug -- Test AUC: ', test_auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "b2b4ff58-b59e-49a8-9ba7-a0d1ab5804b5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-23T12:12:02.001918Z",
     "iopub.status.busy": "2022-05-23T12:12:02.001585Z",
     "iopub.status.idle": "2022-05-23T12:12:04.076539Z",
     "shell.execute_reply": "2022-05-23T12:12:04.076045Z",
     "shell.execute_reply.started": "2022-05-23T12:12:02.001876Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "spark.stop()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
